<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="generator" content="Asciidoctor 2.0.9">
<meta name="author" content="尚硅谷大数据讲师左元">
<title>尚硅谷机器学习和推荐系统教程</title>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Open+Sans:300,300italic,400,400italic,600,600italic%7CNoto+Serif:400,400italic,700,700italic%7CDroid+Sans+Mono:400,700">
<style>
/* Asciidoctor default stylesheet | MIT License | https://asciidoctor.org */
/* Uncomment @import statement to use as custom stylesheet */
/*@import "https://fonts.googleapis.com/css?family=Open+Sans:300,300italic,400,400italic,600,600italic%7CNoto+Serif:400,400italic,700,700italic%7CDroid+Sans+Mono:400,700";*/
article,aside,details,figcaption,figure,footer,header,hgroup,main,nav,section{display:block}
audio,video{display:inline-block}
audio:not([controls]){display:none;height:0}
html{font-family:sans-serif;-ms-text-size-adjust:100%;-webkit-text-size-adjust:100%}
a{background:none}
a:focus{outline:thin dotted}
a:active,a:hover{outline:0}
h1{font-size:2em;margin:.67em 0}
abbr[title]{border-bottom:1px dotted}
b,strong{font-weight:bold}
dfn{font-style:italic}
hr{-moz-box-sizing:content-box;box-sizing:content-box;height:0}
mark{background:#ff0;color:#000}
code,kbd,pre,samp{font-family:monospace;font-size:1em}
pre{white-space:pre-wrap}
q{quotes:"\201C" "\201D" "\2018" "\2019"}
small{font-size:80%}
sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}
sup{top:-.5em}
sub{bottom:-.25em}
img{border:0}
svg:not(:root){overflow:hidden}
figure{margin:0}
fieldset{border:1px solid silver;margin:0 2px;padding:.35em .625em .75em}
legend{border:0;padding:0}
button,input,select,textarea{font-family:inherit;font-size:100%;margin:0}
button,input{line-height:normal}
button,select{text-transform:none}
button,html input[type="button"],input[type="reset"],input[type="submit"]{-webkit-appearance:button;cursor:pointer}
button[disabled],html input[disabled]{cursor:default}
input[type="checkbox"],input[type="radio"]{box-sizing:border-box;padding:0}
button::-moz-focus-inner,input::-moz-focus-inner{border:0;padding:0}
textarea{overflow:auto;vertical-align:top}
table{border-collapse:collapse;border-spacing:0}
*,*::before,*::after{-moz-box-sizing:border-box;-webkit-box-sizing:border-box;box-sizing:border-box}
html,body{font-size:100%}
body{background:#fff;color:rgba(0,0,0,.8);padding:0;margin:0;font-family:"Noto Serif","DejaVu Serif",serif;font-weight:400;font-style:normal;line-height:1;position:relative;cursor:auto;tab-size:4;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased}
a:hover{cursor:pointer}
img,object,embed{max-width:100%;height:auto}
object,embed{height:100%}
img{-ms-interpolation-mode:bicubic}
.left{float:left!important}
.right{float:right!important}
.text-left{text-align:left!important}
.text-right{text-align:right!important}
.text-center{text-align:center!important}
.text-justify{text-align:justify!important}
.hide{display:none}
img,object,svg{display:inline-block;vertical-align:middle}
textarea{height:auto;min-height:50px}
select{width:100%}
.center{margin-left:auto;margin-right:auto}
.stretch{width:100%}
.subheader,.admonitionblock td.content>.title,.audioblock>.title,.exampleblock>.title,.imageblock>.title,.listingblock>.title,.literalblock>.title,.stemblock>.title,.openblock>.title,.paragraph>.title,.quoteblock>.title,table.tableblock>.title,.verseblock>.title,.videoblock>.title,.dlist>.title,.olist>.title,.ulist>.title,.qlist>.title,.hdlist>.title{line-height:1.45;color:#7a2518;font-weight:400;margin-top:0;margin-bottom:.25em}
div,dl,dt,dd,ul,ol,li,h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6,pre,form,p,blockquote,th,td{margin:0;padding:0;direction:ltr}
a{color:#2156a5;text-decoration:underline;line-height:inherit}
a:hover,a:focus{color:#1d4b8f}
a img{border:0}
p{font-family:inherit;font-weight:400;font-size:1em;line-height:1.6;margin-bottom:1.25em;text-rendering:optimizeLegibility}
p aside{font-size:.875em;line-height:1.35;font-style:italic}
h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6{font-family:"Open Sans","DejaVu Sans",sans-serif;font-weight:300;font-style:normal;color:#ba3925;text-rendering:optimizeLegibility;margin-top:1em;margin-bottom:.5em;line-height:1.0125em}
h1 small,h2 small,h3 small,#toctitle small,.sidebarblock>.content>.title small,h4 small,h5 small,h6 small{font-size:60%;color:#e99b8f;line-height:0}
h1{font-size:2.125em}
h2{font-size:1.6875em}
h3,#toctitle,.sidebarblock>.content>.title{font-size:1.375em}
h4,h5{font-size:1.125em}
h6{font-size:1em}
hr{border:solid #dddddf;border-width:1px 0 0;clear:both;margin:1.25em 0 1.1875em;height:0}
em,i{font-style:italic;line-height:inherit}
strong,b{font-weight:bold;line-height:inherit}
small{font-size:60%;line-height:inherit}
code{font-family:"Droid Sans Mono","DejaVu Sans Mono",monospace;font-weight:400;color:rgba(0,0,0,.9)}
ul,ol,dl{font-size:1em;line-height:1.6;margin-bottom:1.25em;list-style-position:outside;font-family:inherit}
ul,ol{margin-left:1.5em}
ul li ul,ul li ol{margin-left:1.25em;margin-bottom:0;font-size:1em}
ul.square li ul,ul.circle li ul,ul.disc li ul{list-style:inherit}
ul.square{list-style-type:square}
ul.circle{list-style-type:circle}
ul.disc{list-style-type:disc}
ol li ul,ol li ol{margin-left:1.25em;margin-bottom:0}
dl dt{margin-bottom:.3125em;font-weight:bold}
dl dd{margin-bottom:1.25em}
abbr,acronym{text-transform:uppercase;font-size:90%;color:rgba(0,0,0,.8);border-bottom:1px dotted #ddd;cursor:help}
abbr{text-transform:none}
blockquote{margin:0 0 1.25em;padding:.5625em 1.25em 0 1.1875em;border-left:1px solid #ddd}
blockquote cite{display:block;font-size:.9375em;color:rgba(0,0,0,.6)}
blockquote cite::before{content:"\2014 \0020"}
blockquote cite a,blockquote cite a:visited{color:rgba(0,0,0,.6)}
blockquote,blockquote p{line-height:1.6;color:rgba(0,0,0,.85)}
@media screen and (min-width:768px){h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6{line-height:1.2}
h1{font-size:2.75em}
h2{font-size:2.3125em}
h3,#toctitle,.sidebarblock>.content>.title{font-size:1.6875em}
h4{font-size:1.4375em}}
table{background:#fff;margin-bottom:1.25em;border:solid 1px #dedede}
table thead,table tfoot{background:#f7f8f7}
table thead tr th,table thead tr td,table tfoot tr th,table tfoot tr td{padding:.5em .625em .625em;font-size:inherit;color:rgba(0,0,0,.8);text-align:left}
table tr th,table tr td{padding:.5625em .625em;font-size:inherit;color:rgba(0,0,0,.8)}
table tr.even,table tr.alt{background:#f8f8f7}
table thead tr th,table tfoot tr th,table tbody tr td,table tr td,table tfoot tr td{display:table-cell;line-height:1.6}
h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6{line-height:1.2;word-spacing:-.05em}
h1 strong,h2 strong,h3 strong,#toctitle strong,.sidebarblock>.content>.title strong,h4 strong,h5 strong,h6 strong{font-weight:400}
.clearfix::before,.clearfix::after,.float-group::before,.float-group::after{content:" ";display:table}
.clearfix::after,.float-group::after{clear:both}
:not(pre):not([class^=L])>code{font-size:.9375em;font-style:normal!important;letter-spacing:0;padding:.1em .5ex;word-spacing:-.15em;background:#f7f7f8;-webkit-border-radius:4px;border-radius:4px;line-height:1.45;text-rendering:optimizeSpeed;word-wrap:break-word}
:not(pre)>code.nobreak{word-wrap:normal}
:not(pre)>code.nowrap{white-space:nowrap}
pre{color:rgba(0,0,0,.9);font-family:"Droid Sans Mono","DejaVu Sans Mono",monospace;line-height:1.45;text-rendering:optimizeSpeed}
pre code,pre pre{color:inherit;font-size:inherit;line-height:inherit}
pre>code{display:block}
pre.nowrap,pre.nowrap pre{white-space:pre;word-wrap:normal}
em em{font-style:normal}
strong strong{font-weight:400}
.keyseq{color:rgba(51,51,51,.8)}
kbd{font-family:"Droid Sans Mono","DejaVu Sans Mono",monospace;display:inline-block;color:rgba(0,0,0,.8);font-size:.65em;line-height:1.45;background:#f7f7f7;border:1px solid #ccc;-webkit-border-radius:3px;border-radius:3px;-webkit-box-shadow:0 1px 0 rgba(0,0,0,.2),0 0 0 .1em white inset;box-shadow:0 1px 0 rgba(0,0,0,.2),0 0 0 .1em #fff inset;margin:0 .15em;padding:.2em .5em;vertical-align:middle;position:relative;top:-.1em;white-space:nowrap}
.keyseq kbd:first-child{margin-left:0}
.keyseq kbd:last-child{margin-right:0}
.menuseq,.menuref{color:#000}
.menuseq b:not(.caret),.menuref{font-weight:inherit}
.menuseq{word-spacing:-.02em}
.menuseq b.caret{font-size:1.25em;line-height:.8}
.menuseq i.caret{font-weight:bold;text-align:center;width:.45em}
b.button::before,b.button::after{position:relative;top:-1px;font-weight:400}
b.button::before{content:"[";padding:0 3px 0 2px}
b.button::after{content:"]";padding:0 2px 0 3px}
p a>code:hover{color:rgba(0,0,0,.9)}
#header,#content,#footnotes,#footer{width:100%;margin-left:auto;margin-right:auto;margin-top:0;margin-bottom:0;max-width:62.5em;*zoom:1;position:relative;padding-left:.9375em;padding-right:.9375em}
#header::before,#header::after,#content::before,#content::after,#footnotes::before,#footnotes::after,#footer::before,#footer::after{content:" ";display:table}
#header::after,#content::after,#footnotes::after,#footer::after{clear:both}
#content{margin-top:1.25em}
#content::before{content:none}
#header>h1:first-child{color:rgba(0,0,0,.85);margin-top:2.25rem;margin-bottom:0}
#header>h1:first-child+#toc{margin-top:8px;border-top:1px solid #dddddf}
#header>h1:only-child,body.toc2 #header>h1:nth-last-child(2){border-bottom:1px solid #dddddf;padding-bottom:8px}
#header .details{border-bottom:1px solid #dddddf;line-height:1.45;padding-top:.25em;padding-bottom:.25em;padding-left:.25em;color:rgba(0,0,0,.6);display:-ms-flexbox;display:-webkit-flex;display:flex;-ms-flex-flow:row wrap;-webkit-flex-flow:row wrap;flex-flow:row wrap}
#header .details span:first-child{margin-left:-.125em}
#header .details span.email a{color:rgba(0,0,0,.85)}
#header .details br{display:none}
#header .details br+span::before{content:"\00a0\2013\00a0"}
#header .details br+span.author::before{content:"\00a0\22c5\00a0";color:rgba(0,0,0,.85)}
#header .details br+span#revremark::before{content:"\00a0|\00a0"}
#header #revnumber{text-transform:capitalize}
#header #revnumber::after{content:"\00a0"}
#content>h1:first-child:not([class]){color:rgba(0,0,0,.85);border-bottom:1px solid #dddddf;padding-bottom:8px;margin-top:0;padding-top:1rem;margin-bottom:1.25rem}
#toc{border-bottom:1px solid #e7e7e9;padding-bottom:.5em}
#toc>ul{margin-left:.125em}
#toc ul.sectlevel0>li>a{font-style:italic}
#toc ul.sectlevel0 ul.sectlevel1{margin:.5em 0}
#toc ul{font-family:"Open Sans","DejaVu Sans",sans-serif;list-style-type:none}
#toc li{line-height:1.3334;margin-top:.3334em}
#toc a{text-decoration:none}
#toc a:active{text-decoration:underline}
#toctitle{color:#7a2518;font-size:1.2em}
@media screen and (min-width:768px){#toctitle{font-size:1.375em}
body.toc2{padding-left:15em;padding-right:0}
#toc.toc2{margin-top:0!important;background:#f8f8f7;position:fixed;width:15em;left:0;top:0;border-right:1px solid #e7e7e9;border-top-width:0!important;border-bottom-width:0!important;z-index:1000;padding:1.25em 1em;height:100%;overflow:auto}
#toc.toc2 #toctitle{margin-top:0;margin-bottom:.8rem;font-size:1.2em}
#toc.toc2>ul{font-size:.9em;margin-bottom:0}
#toc.toc2 ul ul{margin-left:0;padding-left:1em}
#toc.toc2 ul.sectlevel0 ul.sectlevel1{padding-left:0;margin-top:.5em;margin-bottom:.5em}
body.toc2.toc-right{padding-left:0;padding-right:15em}
body.toc2.toc-right #toc.toc2{border-right-width:0;border-left:1px solid #e7e7e9;left:auto;right:0}}
@media screen and (min-width:1280px){body.toc2{padding-left:20em;padding-right:0}
#toc.toc2{width:20em}
#toc.toc2 #toctitle{font-size:1.375em}
#toc.toc2>ul{font-size:.95em}
#toc.toc2 ul ul{padding-left:1.25em}
body.toc2.toc-right{padding-left:0;padding-right:20em}}
#content #toc{border-style:solid;border-width:1px;border-color:#e0e0dc;margin-bottom:1.25em;padding:1.25em;background:#f8f8f7;-webkit-border-radius:4px;border-radius:4px}
#content #toc>:first-child{margin-top:0}
#content #toc>:last-child{margin-bottom:0}
#footer{max-width:100%;background:rgba(0,0,0,.8);padding:1.25em}
#footer-text{color:rgba(255,255,255,.8);line-height:1.44}
#content{margin-bottom:.625em}
.sect1{padding-bottom:.625em}
@media screen and (min-width:768px){#content{margin-bottom:1.25em}
.sect1{padding-bottom:1.25em}}
.sect1:last-child{padding-bottom:0}
.sect1+.sect1{border-top:1px solid #e7e7e9}
#content h1>a.anchor,h2>a.anchor,h3>a.anchor,#toctitle>a.anchor,.sidebarblock>.content>.title>a.anchor,h4>a.anchor,h5>a.anchor,h6>a.anchor{position:absolute;z-index:1001;width:1.5ex;margin-left:-1.5ex;display:block;text-decoration:none!important;visibility:hidden;text-align:center;font-weight:400}
#content h1>a.anchor::before,h2>a.anchor::before,h3>a.anchor::before,#toctitle>a.anchor::before,.sidebarblock>.content>.title>a.anchor::before,h4>a.anchor::before,h5>a.anchor::before,h6>a.anchor::before{content:"\00A7";font-size:.85em;display:block;padding-top:.1em}
#content h1:hover>a.anchor,#content h1>a.anchor:hover,h2:hover>a.anchor,h2>a.anchor:hover,h3:hover>a.anchor,#toctitle:hover>a.anchor,.sidebarblock>.content>.title:hover>a.anchor,h3>a.anchor:hover,#toctitle>a.anchor:hover,.sidebarblock>.content>.title>a.anchor:hover,h4:hover>a.anchor,h4>a.anchor:hover,h5:hover>a.anchor,h5>a.anchor:hover,h6:hover>a.anchor,h6>a.anchor:hover{visibility:visible}
#content h1>a.link,h2>a.link,h3>a.link,#toctitle>a.link,.sidebarblock>.content>.title>a.link,h4>a.link,h5>a.link,h6>a.link{color:#ba3925;text-decoration:none}
#content h1>a.link:hover,h2>a.link:hover,h3>a.link:hover,#toctitle>a.link:hover,.sidebarblock>.content>.title>a.link:hover,h4>a.link:hover,h5>a.link:hover,h6>a.link:hover{color:#a53221}
details,.audioblock,.imageblock,.literalblock,.listingblock,.stemblock,.videoblock{margin-bottom:1.25em}
details>summary:first-of-type{cursor:pointer;display:list-item;outline:none;margin-bottom:.75em}
.admonitionblock td.content>.title,.audioblock>.title,.exampleblock>.title,.imageblock>.title,.listingblock>.title,.literalblock>.title,.stemblock>.title,.openblock>.title,.paragraph>.title,.quoteblock>.title,table.tableblock>.title,.verseblock>.title,.videoblock>.title,.dlist>.title,.olist>.title,.ulist>.title,.qlist>.title,.hdlist>.title{text-rendering:optimizeLegibility;text-align:left;font-family:"Noto Serif","DejaVu Serif",serif;font-size:1rem;font-style:italic}
table.tableblock.fit-content>caption.title{white-space:nowrap;width:0}
.paragraph.lead>p,#preamble>.sectionbody>[class="paragraph"]:first-of-type p{font-size:1.21875em;line-height:1.6;color:rgba(0,0,0,.85)}
table.tableblock #preamble>.sectionbody>[class="paragraph"]:first-of-type p{font-size:inherit}
.admonitionblock>table{border-collapse:separate;border:0;background:none;width:100%}
.admonitionblock>table td.icon{text-align:center;width:80px}
.admonitionblock>table td.icon img{max-width:none}
.admonitionblock>table td.icon .title{font-weight:bold;font-family:"Open Sans","DejaVu Sans",sans-serif;text-transform:uppercase}
.admonitionblock>table td.content{padding-left:1.125em;padding-right:1.25em;border-left:1px solid #dddddf;color:rgba(0,0,0,.6)}
.admonitionblock>table td.content>:last-child>:last-child{margin-bottom:0}
.exampleblock>.content{border-style:solid;border-width:1px;border-color:#e6e6e6;margin-bottom:1.25em;padding:1.25em;background:#fff;-webkit-border-radius:4px;border-radius:4px}
.exampleblock>.content>:first-child{margin-top:0}
.exampleblock>.content>:last-child{margin-bottom:0}
.sidebarblock{border-style:solid;border-width:1px;border-color:#dbdbd6;margin-bottom:1.25em;padding:1.25em;background:#f3f3f2;-webkit-border-radius:4px;border-radius:4px}
.sidebarblock>:first-child{margin-top:0}
.sidebarblock>:last-child{margin-bottom:0}
.sidebarblock>.content>.title{color:#7a2518;margin-top:0;text-align:center}
.exampleblock>.content>:last-child>:last-child,.exampleblock>.content .olist>ol>li:last-child>:last-child,.exampleblock>.content .ulist>ul>li:last-child>:last-child,.exampleblock>.content .qlist>ol>li:last-child>:last-child,.sidebarblock>.content>:last-child>:last-child,.sidebarblock>.content .olist>ol>li:last-child>:last-child,.sidebarblock>.content .ulist>ul>li:last-child>:last-child,.sidebarblock>.content .qlist>ol>li:last-child>:last-child{margin-bottom:0}
.literalblock pre,.listingblock>.content>pre{-webkit-border-radius:4px;border-radius:4px;word-wrap:break-word;overflow-x:auto;padding:1em;font-size:.8125em}
@media screen and (min-width:768px){.literalblock pre,.listingblock>.content>pre{font-size:.90625em}}
@media screen and (min-width:1280px){.literalblock pre,.listingblock>.content>pre{font-size:1em}}
.literalblock pre,.listingblock>.content>pre:not(.highlight),.listingblock>.content>pre[class="highlight"],.listingblock>.content>pre[class^="highlight "]{background:#f7f7f8}
.literalblock.output pre{color:#f7f7f8;background:rgba(0,0,0,.9)}
.listingblock>.content{position:relative}
.listingblock code[data-lang]::before{display:none;content:attr(data-lang);position:absolute;font-size:.75em;top:.425rem;right:.5rem;line-height:1;text-transform:uppercase;color:inherit;opacity:.5}
.listingblock:hover code[data-lang]::before{display:block}
.listingblock.terminal pre .command::before{content:attr(data-prompt);padding-right:.5em;color:inherit;opacity:.5}
.listingblock.terminal pre .command:not([data-prompt])::before{content:"$"}
.listingblock pre.highlightjs{padding:0}
.listingblock pre.highlightjs>code{padding:1em;-webkit-border-radius:4px;border-radius:4px}
.listingblock pre.prettyprint{border-width:0}
.prettyprint{background:#f7f7f8}
pre.prettyprint .linenums{line-height:1.45;margin-left:2em}
pre.prettyprint li{background:none;list-style-type:inherit;padding-left:0}
pre.prettyprint li code[data-lang]::before{opacity:1}
pre.prettyprint li:not(:first-child) code[data-lang]::before{display:none}
table.linenotable{border-collapse:separate;border:0;margin-bottom:0;background:none}
table.linenotable td[class]{color:inherit;vertical-align:top;padding:0;line-height:inherit;white-space:normal}
table.linenotable td.code{padding-left:.75em}
table.linenotable td.linenos{border-right:1px solid currentColor;opacity:.35;padding-right:.5em}
pre.pygments .lineno{border-right:1px solid currentColor;opacity:.35;display:inline-block;margin-right:.75em}
pre.pygments .lineno::before{content:"";margin-right:-.125em}
.quoteblock{margin:0 1em 1.25em 1.5em;display:table}
.quoteblock>.title{margin-left:-1.5em;margin-bottom:.75em}
.quoteblock blockquote,.quoteblock p{color:rgba(0,0,0,.85);font-size:1.15rem;line-height:1.75;word-spacing:.1em;letter-spacing:0;font-style:italic;text-align:justify}
.quoteblock blockquote{margin:0;padding:0;border:0}
.quoteblock blockquote::before{content:"\201c";float:left;font-size:2.75em;font-weight:bold;line-height:.6em;margin-left:-.6em;color:#7a2518;text-shadow:0 1px 2px rgba(0,0,0,.1)}
.quoteblock blockquote>.paragraph:last-child p{margin-bottom:0}
.quoteblock .attribution{margin-top:.75em;margin-right:.5ex;text-align:right}
.verseblock{margin:0 1em 1.25em}
.verseblock pre{font-family:"Open Sans","DejaVu Sans",sans;font-size:1.15rem;color:rgba(0,0,0,.85);font-weight:300;text-rendering:optimizeLegibility}
.verseblock pre strong{font-weight:400}
.verseblock .attribution{margin-top:1.25rem;margin-left:.5ex}
.quoteblock .attribution,.verseblock .attribution{font-size:.9375em;line-height:1.45;font-style:italic}
.quoteblock .attribution br,.verseblock .attribution br{display:none}
.quoteblock .attribution cite,.verseblock .attribution cite{display:block;letter-spacing:-.025em;color:rgba(0,0,0,.6)}
.quoteblock.abstract blockquote::before,.quoteblock.excerpt blockquote::before,.quoteblock .quoteblock blockquote::before{display:none}
.quoteblock.abstract blockquote,.quoteblock.abstract p,.quoteblock.excerpt blockquote,.quoteblock.excerpt p,.quoteblock .quoteblock blockquote,.quoteblock .quoteblock p{line-height:1.6;word-spacing:0}
.quoteblock.abstract{margin:0 1em 1.25em;display:block}
.quoteblock.abstract>.title{margin:0 0 .375em;font-size:1.15em;text-align:center}
.quoteblock.excerpt,.quoteblock .quoteblock{margin:0 0 1.25em;padding:0 0 .25em 1em;border-left:.25em solid #dddddf}
.quoteblock.excerpt blockquote,.quoteblock.excerpt p,.quoteblock .quoteblock blockquote,.quoteblock .quoteblock p{color:inherit;font-size:1.0625rem}
.quoteblock.excerpt .attribution,.quoteblock .quoteblock .attribution{color:inherit;text-align:left;margin-right:0}
table.tableblock{max-width:100%;border-collapse:separate}
p.tableblock:last-child{margin-bottom:0}
td.tableblock>.content>:last-child{margin-bottom:-1.25em}
td.tableblock>.content>:last-child.sidebarblock{margin-bottom:0}
table.tableblock,th.tableblock,td.tableblock{border:0 solid #dedede}
table.grid-all>thead>tr>.tableblock,table.grid-all>tbody>tr>.tableblock{border-width:0 1px 1px 0}
table.grid-all>tfoot>tr>.tableblock{border-width:1px 1px 0 0}
table.grid-cols>*>tr>.tableblock{border-width:0 1px 0 0}
table.grid-rows>thead>tr>.tableblock,table.grid-rows>tbody>tr>.tableblock{border-width:0 0 1px}
table.grid-rows>tfoot>tr>.tableblock{border-width:1px 0 0}
table.grid-all>*>tr>.tableblock:last-child,table.grid-cols>*>tr>.tableblock:last-child{border-right-width:0}
table.grid-all>tbody>tr:last-child>.tableblock,table.grid-all>thead:last-child>tr>.tableblock,table.grid-rows>tbody>tr:last-child>.tableblock,table.grid-rows>thead:last-child>tr>.tableblock{border-bottom-width:0}
table.frame-all{border-width:1px}
table.frame-sides{border-width:0 1px}
table.frame-topbot,table.frame-ends{border-width:1px 0}
table.stripes-all tr,table.stripes-odd tr:nth-of-type(odd),table.stripes-even tr:nth-of-type(even),table.stripes-hover tr:hover{background:#f8f8f7}
th.halign-left,td.halign-left{text-align:left}
th.halign-right,td.halign-right{text-align:right}
th.halign-center,td.halign-center{text-align:center}
th.valign-top,td.valign-top{vertical-align:top}
th.valign-bottom,td.valign-bottom{vertical-align:bottom}
th.valign-middle,td.valign-middle{vertical-align:middle}
table thead th,table tfoot th{font-weight:bold}
tbody tr th{display:table-cell;line-height:1.6;background:#f7f8f7}
tbody tr th,tbody tr th p,tfoot tr th,tfoot tr th p{color:rgba(0,0,0,.8);font-weight:bold}
p.tableblock>code:only-child{background:none;padding:0}
p.tableblock{font-size:1em}
ol{margin-left:1.75em}
ul li ol{margin-left:1.5em}
dl dd{margin-left:1.125em}
dl dd:last-child,dl dd:last-child>:last-child{margin-bottom:0}
ol>li p,ul>li p,ul dd,ol dd,.olist .olist,.ulist .ulist,.ulist .olist,.olist .ulist{margin-bottom:.625em}
ul.checklist,ul.none,ol.none,ul.no-bullet,ol.no-bullet,ol.unnumbered,ul.unstyled,ol.unstyled{list-style-type:none}
ul.no-bullet,ol.no-bullet,ol.unnumbered{margin-left:.625em}
ul.unstyled,ol.unstyled{margin-left:0}
ul.checklist{margin-left:.625em}
ul.checklist li>p:first-child>.fa-square-o:first-child,ul.checklist li>p:first-child>.fa-check-square-o:first-child{width:1.25em;font-size:.8em;position:relative;bottom:.125em}
ul.checklist li>p:first-child>input[type="checkbox"]:first-child{margin-right:.25em}
ul.inline{display:-ms-flexbox;display:-webkit-box;display:flex;-ms-flex-flow:row wrap;-webkit-flex-flow:row wrap;flex-flow:row wrap;list-style:none;margin:0 0 .625em -1.25em}
ul.inline>li{margin-left:1.25em}
.unstyled dl dt{font-weight:400;font-style:normal}
ol.arabic{list-style-type:decimal}
ol.decimal{list-style-type:decimal-leading-zero}
ol.loweralpha{list-style-type:lower-alpha}
ol.upperalpha{list-style-type:upper-alpha}
ol.lowerroman{list-style-type:lower-roman}
ol.upperroman{list-style-type:upper-roman}
ol.lowergreek{list-style-type:lower-greek}
.hdlist>table,.colist>table{border:0;background:none}
.hdlist>table>tbody>tr,.colist>table>tbody>tr{background:none}
td.hdlist1,td.hdlist2{vertical-align:top;padding:0 .625em}
td.hdlist1{font-weight:bold;padding-bottom:1.25em}
.literalblock+.colist,.listingblock+.colist{margin-top:-.5em}
.colist td:not([class]):first-child{padding:.4em .75em 0;line-height:1;vertical-align:top}
.colist td:not([class]):first-child img{max-width:none}
.colist td:not([class]):last-child{padding:.25em 0}
.thumb,.th{line-height:0;display:inline-block;border:solid 4px #fff;-webkit-box-shadow:0 0 0 1px #ddd;box-shadow:0 0 0 1px #ddd}
.imageblock.left{margin:.25em .625em 1.25em 0}
.imageblock.right{margin:.25em 0 1.25em .625em}
.imageblock>.title{margin-bottom:0}
.imageblock.thumb,.imageblock.th{border-width:6px}
.imageblock.thumb>.title,.imageblock.th>.title{padding:0 .125em}
.image.left,.image.right{margin-top:.25em;margin-bottom:.25em;display:inline-block;line-height:0}
.image.left{margin-right:.625em}
.image.right{margin-left:.625em}
a.image{text-decoration:none;display:inline-block}
a.image object{pointer-events:none}
sup.footnote,sup.footnoteref{font-size:.875em;position:static;vertical-align:super}
sup.footnote a,sup.footnoteref a{text-decoration:none}
sup.footnote a:active,sup.footnoteref a:active{text-decoration:underline}
#footnotes{padding-top:.75em;padding-bottom:.75em;margin-bottom:.625em}
#footnotes hr{width:20%;min-width:6.25em;margin:-.25em 0 .75em;border-width:1px 0 0}
#footnotes .footnote{padding:0 .375em 0 .225em;line-height:1.3334;font-size:.875em;margin-left:1.2em;margin-bottom:.2em}
#footnotes .footnote a:first-of-type{font-weight:bold;text-decoration:none;margin-left:-1.05em}
#footnotes .footnote:last-of-type{margin-bottom:0}
#content #footnotes{margin-top:-.625em;margin-bottom:0;padding:.75em 0}
.gist .file-data>table{border:0;background:#fff;width:100%;margin-bottom:0}
.gist .file-data>table td.line-data{width:99%}
div.unbreakable{page-break-inside:avoid}
.big{font-size:larger}
.small{font-size:smaller}
.underline{text-decoration:underline}
.overline{text-decoration:overline}
.line-through{text-decoration:line-through}
.aqua{color:#00bfbf}
.aqua-background{background:#00fafa}
.black{color:#000}
.black-background{background:#000}
.blue{color:#0000bf}
.blue-background{background:#0000fa}
.fuchsia{color:#bf00bf}
.fuchsia-background{background:#fa00fa}
.gray{color:#606060}
.gray-background{background:#7d7d7d}
.green{color:#006000}
.green-background{background:#007d00}
.lime{color:#00bf00}
.lime-background{background:#00fa00}
.maroon{color:#600000}
.maroon-background{background:#7d0000}
.navy{color:#000060}
.navy-background{background:#00007d}
.olive{color:#606000}
.olive-background{background:#7d7d00}
.purple{color:#600060}
.purple-background{background:#7d007d}
.red{color:#bf0000}
.red-background{background:#fa0000}
.silver{color:#909090}
.silver-background{background:#bcbcbc}
.teal{color:#006060}
.teal-background{background:#007d7d}
.white{color:#bfbfbf}
.white-background{background:#fafafa}
.yellow{color:#bfbf00}
.yellow-background{background:#fafa00}
span.icon>.fa{cursor:default}
a span.icon>.fa{cursor:inherit}
.admonitionblock td.icon [class^="fa icon-"]{font-size:2.5em;text-shadow:1px 1px 2px rgba(0,0,0,.5);cursor:default}
.admonitionblock td.icon .icon-note::before{content:"\f05a";color:#19407c}
.admonitionblock td.icon .icon-tip::before{content:"\f0eb";text-shadow:1px 1px 2px rgba(155,155,0,.8);color:#111}
.admonitionblock td.icon .icon-warning::before{content:"\f071";color:#bf6900}
.admonitionblock td.icon .icon-caution::before{content:"\f06d";color:#bf3400}
.admonitionblock td.icon .icon-important::before{content:"\f06a";color:#bf0000}
.conum[data-value]{display:inline-block;color:#fff!important;background:rgba(0,0,0,.8);-webkit-border-radius:100px;border-radius:100px;text-align:center;font-size:.75em;width:1.67em;height:1.67em;line-height:1.67em;font-family:"Open Sans","DejaVu Sans",sans-serif;font-style:normal;font-weight:bold}
.conum[data-value] *{color:#fff!important}
.conum[data-value]+b{display:none}
.conum[data-value]::after{content:attr(data-value)}
pre .conum[data-value]{position:relative;top:-.125em}
b.conum *{color:inherit!important}
.conum:not([data-value]):empty{display:none}
dt,th.tableblock,td.content,div.footnote{text-rendering:optimizeLegibility}
h1,h2,p,td.content,span.alt{letter-spacing:-.01em}
p strong,td.content strong,div.footnote strong{letter-spacing:-.005em}
p,blockquote,dt,td.content,span.alt{font-size:1.0625rem}
p{margin-bottom:1.25rem}
.sidebarblock p,.sidebarblock dt,.sidebarblock td.content,p.tableblock{font-size:1em}
.exampleblock>.content{background:#fffef7;border-color:#e0e0dc;-webkit-box-shadow:0 1px 4px #e0e0dc;box-shadow:0 1px 4px #e0e0dc}
.print-only{display:none!important}
@page{margin:1.25cm .75cm}
@media print{*{-webkit-box-shadow:none!important;box-shadow:none!important;text-shadow:none!important}
html{font-size:80%}
a{color:inherit!important;text-decoration:underline!important}
a.bare,a[href^="#"],a[href^="mailto:"]{text-decoration:none!important}
a[href^="http:"]:not(.bare)::after,a[href^="https:"]:not(.bare)::after{content:"(" attr(href) ")";display:inline-block;font-size:.875em;padding-left:.25em}
abbr[title]::after{content:" (" attr(title) ")"}
pre,blockquote,tr,img,object,svg{page-break-inside:avoid}
thead{display:table-header-group}
svg{max-width:100%}
p,blockquote,dt,td.content{font-size:1em;orphans:3;widows:3}
h2,h3,#toctitle,.sidebarblock>.content>.title{page-break-after:avoid}
#toc,.sidebarblock,.exampleblock>.content{background:none!important}
#toc{border-bottom:1px solid #dddddf!important;padding-bottom:0!important}
body.book #header{text-align:center}
body.book #header>h1:first-child{border:0!important;margin:2.5em 0 1em}
body.book #header .details{border:0!important;display:block;padding:0!important}
body.book #header .details span:first-child{margin-left:0!important}
body.book #header .details br{display:block}
body.book #header .details br+span::before{content:none!important}
body.book #toc{border:0!important;text-align:left!important;padding:0!important;margin:0!important}
body.book #toc,body.book #preamble,body.book h1.sect0,body.book .sect1>h2{page-break-before:always}
.listingblock code[data-lang]::before{display:block}
#footer{padding:0 .9375em}
.hide-on-print{display:none!important}
.print-only{display:block!important}
.hide-for-print{display:none!important}
.show-for-print{display:inherit!important}}
@media print,amzn-kf8{#header>h1:first-child{margin-top:1.25rem}
.sect1{padding:0!important}
.sect1+.sect1{border:0}
#footer{background:none}
#footer-text{color:rgba(0,0,0,.6);font-size:.9em}}
@media amzn-kf8{#header,#content,#footnotes,#footer{padding:0}}
</style>
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">
</head>
<body class="article toc2 toc-left">
<div id="header">
<h1>尚硅谷机器学习和推荐系统教程</h1>
<div class="details">
<span id="author" class="author">尚硅谷大数据讲师左元</span><br>
<span id="email" class="email"><a href="mailto:zuoyuan@atguigu.com">zuoyuan@atguigu.com</a></span><br>
<span id="revnumber">version 1.0,</span>
<span id="revdate">2019-06-01</span>
</div>
<div id="toc" class="toc2">
<div id="toctitle">Table of Contents</div>
<ul class="sectlevel1">
<li><a href="#_机器学习数学基础">1. 机器学习数学基础</a>
<ul class="sectlevel2">
<li><a href="#_线性代数">1.1. 线性代数</a>
<ul class="sectlevel3">
<li><a href="#_什么是矩阵">1.1.1. 什么是矩阵</a></li>
<li><a href="#_矩阵">1.1.2. 矩阵</a>
<ul class="sectlevel4">
<li><a href="#_矩阵的定义">矩阵的定义</a></li>
<li><a href="#_特殊矩阵">特殊矩阵</a></li>
</ul>
</li>
<li><a href="#_矩阵中的概念">1.1.3. 矩阵中的概念</a>
<ul class="sectlevel4">
<li><a href="#_特殊矩阵_2">特殊矩阵</a></li>
</ul>
</li>
<li><a href="#_矩阵的加法">1.1.4. 矩阵的加法</a></li>
<li><a href="#_矩阵的乘法">1.1.5. 矩阵的乘法</a>
<ul class="sectlevel4">
<li><a href="#_数与矩阵相乘">数与矩阵相乘</a></li>
<li><a href="#_矩阵与矩阵相乘">矩阵与矩阵相乘</a></li>
</ul>
</li>
<li><a href="#_矩阵的转置">1.1.6. 矩阵的转置</a></li>
<li><a href="#_矩阵的运算方法">1.1.7. 矩阵的运算方法</a>
<ul class="sectlevel4">
<li><a href="#_加法">加法</a></li>
<li><a href="#_乘法">乘法</a></li>
<li><a href="#_减法">减法</a></li>
<li><a href="#_转置">转置</a></li>
</ul>
</li>
<li><a href="#_矩阵的逆">1.1.8. 矩阵的逆</a></li>
</ul>
</li>
<li><a href="#_微积分基础知识">1.2. 微积分基础知识</a>
<ul class="sectlevel3">
<li><a href="#_什么是导数">1.2.1. 什么是导数</a></li>
<li><a href="#_偏导数">1.2.2. 偏导数</a></li>
<li><a href="#_方向导数">1.2.3. 方向导数</a></li>
<li><a href="#_梯度gradient">1.2.4. 梯度（Gradient）</a></li>
<li><a href="#_凸函数和凹函数">1.2.5. 凸函数和凹函数</a>
<ul class="sectlevel4">
<li><a href="#_凸函数">凸函数</a></li>
<li><a href="#_凹函数">凹函数</a></li>
</ul>
</li>
</ul>
</li>
<li><a href="#_概率论和统计学">1.3. 概率论和统计学</a>
<ul class="sectlevel3">
<li><a href="#_常用统计变量">1.3.1. 常用统计变量</a></li>
<li><a href="#_常见概率分布">1.3.2. 常见概率分布</a></li>
<li><a href="#_重要概率公式">1.3.3. 重要概率公式</a></li>
</ul>
</li>
</ul>
</li>
<li><a href="#_数学基础精简进阶版">2. 数学基础(精简进阶版)</a>
<ul class="sectlevel2">
<li><a href="#_线性代数_2">2.1. 线性代数</a>
<ul class="sectlevel3">
<li><a href="#_向量">2.1.1. 向量</a></li>
<li><a href="#_矩阵_2">2.1.2. 矩阵</a></li>
<li><a href="#_运算">2.1.3. 运算</a></li>
<li><a href="#_范数">2.1.4. 范数</a></li>
<li><a href="#_特征向量和特征值">2.1.5. 特征向量和特征值</a></li>
</ul>
</li>
<li><a href="#_微分">2.2. 微分</a>
<ul class="sectlevel3">
<li><a href="#_导数和微分">2.2.1. 导数和微分</a></li>
<li><a href="#_泰勒展开">2.2.2. 泰勒展开</a></li>
<li><a href="#_偏导数_2">2.2.3. 偏导数</a></li>
<li><a href="#_梯度">2.2.4. 梯度</a></li>
<li><a href="#_海森矩阵">2.2.5. 海森矩阵</a></li>
</ul>
</li>
<li><a href="#_概率">2.3. 概率</a>
<ul class="sectlevel3">
<li><a href="#_条件概率">2.3.1. 条件概率</a></li>
<li><a href="#_期望">2.3.2. 期望</a></li>
<li><a href="#_均匀分布">2.3.3. 均匀分布</a></li>
</ul>
</li>
</ul>
</li>
<li><a href="#_机器学习简介">3. 机器学习简介</a>
<ul class="sectlevel2">
<li><a href="#_机器学习的概念">3.1. 机器学习的概念</a>
<ul class="sectlevel3">
<li><a href="#_机器学习是什么">3.1.1. 机器学习是什么</a></li>
<li><a href="#_机器学习的开端">3.1.2. 机器学习的开端</a></li>
<li><a href="#_机器学习的定义">3.1.3. 机器学习的定义</a></li>
<li><a href="#_机器学习的过程">3.1.4. 机器学习的过程</a></li>
</ul>
</li>
<li><a href="#_机器学习的分类">3.2. 机器学习的分类</a>
<ul class="sectlevel3">
<li><a href="#_监督学习">3.2.1. 监督学习</a></li>
<li><a href="#_监督学习深入介绍">3.2.2. 监督学习深入介绍</a>
<ul class="sectlevel4">
<li><a href="#_监督学习三要素">监督学习三要素</a></li>
<li><a href="#_监督学习实现步骤">监督学习实现步骤</a></li>
<li><a href="#_监督学习过程示例">监督学习过程示例</a></li>
</ul>
</li>
<li><a href="#_模型评估策略">3.2.3. 模型评估策略</a>
<ul class="sectlevel4">
<li><a href="#_训练集和测试集">训练集和测试集</a></li>
<li><a href="#_损失函数">损失函数</a></li>
<li><a href="#_常见损失函数">常见损失函数</a></li>
<li><a href="#_经验风险">经验风险</a></li>
<li><a href="#_训练误差和测试误差">训练误差和测试误差</a></li>
<li><a href="#_过拟合和欠拟合">过拟合和欠拟合</a></li>
</ul>
</li>
<li><a href="#_模型的选择">3.2.4. 模型的选择</a></li>
<li><a href="#_正则化">3.2.5. 正则化</a>
<ul class="sectlevel4">
<li><a href="#_奥卡姆剃刀">奥卡姆剃刀</a></li>
</ul>
</li>
<li><a href="#_交叉验证">3.2.6. 交叉验证</a></li>
<li><a href="#_分类和回归">3.2.7. 分类和回归</a>
<ul class="sectlevel4">
<li><a href="#_分类问题">分类问题</a></li>
<li><a href="#_精确率和召回率">精确率和召回率</a></li>
<li><a href="#_回归问题">回归问题</a></li>
</ul>
</li>
<li><a href="#_模型求解算法学习算法">3.2.8. 模型求解算法（学习算法）</a>
<ul class="sectlevel4">
<li><a href="#_梯度下降算法">梯度下降算法</a></li>
</ul>
</li>
<li><a href="#_牛顿法和拟牛顿法">3.2.9. 牛顿法和拟牛顿法</a>
<ul class="sectlevel4">
<li><a href="#_牛顿法">牛顿法</a></li>
<li><a href="#_拟牛顿法">拟牛顿法</a></li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li><a href="#_机器学习模型简介">4. 机器学习模型简介</a>
<ul class="sectlevel2">
<li><a href="#_使用anaconda">4.1. 使用Anaconda</a></li>
<li><a href="#_线性回归">4.2. 线性回归</a>
<ul class="sectlevel3">
<li><a href="#_损失函数_2">4.2.1. 损失函数</a></li>
<li><a href="#_学习算法">4.2.2. 学习算法</a></li>
<li><a href="#_最小二乘法">4.2.3. 最小二乘法</a>
<ul class="sectlevel4">
<li><a href="#_最小二乘法求解过程">最小二乘法求解过程</a></li>
</ul>
</li>
<li><a href="#_梯度下降法求解过程">4.2.4. 梯度下降法求解过程</a></li>
<li><a href="#_梯度下降法和最小二乘法">4.2.5. 梯度下降法和最小二乘法</a></li>
</ul>
</li>
<li><a href="#_感知机perceptron">4.3. 感知机（Perceptron）</a>
<ul class="sectlevel3">
<li><a href="#_感知机的定义">4.3.1. 感知机的定义</a></li>
<li><a href="#_感知机的数学原理">4.3.2. 感知机的数学原理</a></li>
<li><a href="#_感知机的模型">4.3.3. 感知机的模型</a></li>
<li><a href="#_感知机的损失函数">4.3.4. 感知机的损失函数</a></li>
<li><a href="#_感知机学习算法">4.3.5. 感知机学习算法</a></li>
</ul>
</li>
<li><a href="#_逻辑回归logistic_regression">4.4. 逻辑回归（Logistic Regression）</a>
<ul class="sectlevel3">
<li><a href="#_直观理解">4.4.1. 直观理解</a></li>
<li><a href="#_逻辑回归模型">4.4.2. 逻辑回归模型</a></li>
<li><a href="#_损失函数_3">4.4.3. 损失函数</a></li>
<li><a href="#_梯度下降">4.4.4. 梯度下降</a></li>
</ul>
</li>
<li><a href="#_knn">4.5. KNN</a>
<ul class="sectlevel3">
<li><a href="#_knn例子">4.5.1. KNN例子</a></li>
<li><a href="#_knn距离计算">4.5.2. KNN距离计算</a></li>
<li><a href="#_knn算法">4.5.3. KNN算法</a></li>
</ul>
</li>
<li><a href="#_决策树">4.6. 决策树</a></li>
</ul>
</li>
<li><a href="#_推荐系统项目实战">5. 推荐系统项目实战</a>
<ul class="sectlevel2">
<li><a href="#_项目体系架构设计">5.1. 项目体系架构设计</a>
<ul class="sectlevel3">
<li><a href="#_项目系统架构">5.1.1. 项目系统架构</a></li>
<li><a href="#_数据存储部分">5.1.2. 数据存储部分</a>
<ul class="sectlevel4">
<li><a href="#_离线推荐部分">离线推荐部分</a></li>
<li><a href="#_实时推荐部分">实时推荐部分</a></li>
</ul>
</li>
</ul>
</li>
<li><a href="#_项目数据流程">5.2. 项目数据流程</a>
<ul class="sectlevel3">
<li><a href="#_系统初始化部分">5.2.1. 系统初始化部分</a></li>
<li><a href="#_实时推荐部分_2">5.2.2. 实时推荐部分</a></li>
<li><a href="#_业务系统部分">5.2.3. 业务系统部分</a></li>
</ul>
</li>
<li><a href="#_数据模型">5.3. 数据模型</a></li>
<li><a href="#_工具环境搭建">5.4. 工具环境搭建</a>
<ul class="sectlevel3">
<li><a href="#_mongodb单节点环境配置">5.4.1. MongoDB（单节点）环境配置</a></li>
<li><a href="#_redis单节点环境配置">5.4.2. Redis（单节点）环境配置</a></li>
<li><a href="#_spark单节点环境配置">5.4.3. Spark（单节点）环境配置</a></li>
<li><a href="#_zookeeper单节点环境配置">5.4.4. Zookeeper（单节点）环境配置</a></li>
<li><a href="#_flume_ng单节点环境配置">5.4.5. Flume-ng（单节点）环境配置</a></li>
<li><a href="#_kafka单节点环境配置">5.4.6. Kafka（单节点）环境配置</a></li>
</ul>
</li>
<li><a href="#_创建项目并初始化业务数据">5.5. 创建项目并初始化业务数据</a>
<ul class="sectlevel3">
<li><a href="#_在idea中创建maven项目">5.5.1. 在IDEA中创建maven项目</a></li>
<li><a href="#_项目框架搭建">5.5.2. 项目框架搭建</a></li>
<li><a href="#_声明项目中工具的版本信息">5.5.3. 声明项目中工具的版本信息</a></li>
<li><a href="#_添加项目依赖">5.5.4. 添加项目依赖</a></li>
</ul>
</li>
<li><a href="#_数据加载准备">5.6. 数据加载准备</a>
<ul class="sectlevel3">
<li><a href="#_products数据集">5.6.1. Products数据集</a></li>
<li><a href="#_ratings数据集">5.6.2. Ratings数据集</a></li>
<li><a href="#_日志管理配置文件">5.6.3. 日志管理配置文件</a></li>
</ul>
</li>
<li><a href="#_数据初始化到mongodb">5.7. 数据初始化到MongoDB</a>
<ul class="sectlevel3">
<li><a href="#_启动mongodb数据库略">5.7.1. 启动MongoDB数据库（略）</a></li>
<li><a href="#_数据加载程序主体实现">5.7.2. 数据加载程序主体实现</a></li>
</ul>
</li>
<li><a href="#_离线推荐服务建设">5.8. 离线推荐服务建设</a>
<ul class="sectlevel3">
<li><a href="#_离线推荐服务">5.8.1. 离线推荐服务</a></li>
<li><a href="#_离线统计服务">5.8.2. 离线统计服务</a>
<ul class="sectlevel4">
<li><a href="#_历史热门商品统计">历史热门商品统计</a></li>
<li><a href="#_最近热门商品统计">最近热门商品统计</a></li>
<li><a href="#_商品平均得分统计">商品平均得分统计</a></li>
</ul>
</li>
<li><a href="#_基于隐语义模型的协同过滤推荐">5.8.3. 基于隐语义模型的协同过滤推荐</a>
<ul class="sectlevel4">
<li><a href="#_用户商品推荐矩阵">用户商品推荐矩阵</a></li>
<li><a href="#_商品相似度矩阵">商品相似度矩阵</a></li>
<li><a href="#_模型评估和参数选取">模型评估和参数选取</a></li>
</ul>
</li>
</ul>
</li>
<li><a href="#_实时推荐服务建设">5.9. 实时推荐服务建设</a>
<ul class="sectlevel3">
<li><a href="#_实时推荐服务">5.9.1. 实时推荐服务</a></li>
<li><a href="#_实时推荐算法设计">5.9.2. 实时推荐算法设计</a></li>
<li><a href="#_实时推荐算法的讲解">5.9.3. 实时推荐算法的讲解</a>
<ul class="sectlevel4">
<li><a href="#_获取用户的k次最近评分">获取用户的K次最近评分</a></li>
<li><a href="#_获取当前商品最相似的k个商品">获取当前商品最相似的K个商品</a></li>
<li><a href="#_商品推荐优先级计算">商品推荐优先级计算</a></li>
<li><a href="#_将结果保存到mongodb">将结果保存到mongoDB</a></li>
<li><a href="#_更新实时推荐结果">更新实时推荐结果</a></li>
</ul>
</li>
<li><a href="#_实时系统联调">5.9.4. 实时系统联调</a>
<ul class="sectlevel4">
<li><a href="#_启动实时系统的基本组件">启动实时系统的基本组件</a></li>
<li><a href="#_启动zookeeper">启动zookeeper</a></li>
<li><a href="#_启动kafka">启动kafka</a></li>
<li><a href="#_构建kafka_streaming程序">构建Kafka Streaming程序</a></li>
<li><a href="#_配置并启动flume">配置并启动flume</a></li>
<li><a href="#_启动业务系统后台">启动业务系统后台</a></li>
</ul>
</li>
</ul>
</li>
<li><a href="#_冷启动问题处理">5.10. 冷启动问题处理</a></li>
<li><a href="#_基于内容的推荐服务">5.11. 基于内容的推荐服务</a>
<ul class="sectlevel3">
<li><a href="#_基于内容的推荐服务_2">5.11.1. 基于内容的推荐服务</a></li>
<li><a href="#_基于内容推荐的实现">5.11.2. 基于内容推荐的实现</a></li>
</ul>
</li>
<li><a href="#_基于物品的协同过滤">5.12. 基于物品的协同过滤</a></li>
<li><a href="#_附记">5.13. 附记</a></li>
<li><a href="#_算法使用场景">5.14. 算法使用场景：</a></li>
</ul>
</li>
<li><a href="#_用python从头实现神经网络">6. 用Python从头实现神经网络</a>
<ul class="sectlevel2">
<li><a href="#_砖块神经元">6.1. 砖块：神经元</a></li>
<li><a href="#_一个简单的例子">6.2. 一个简单的例子</a></li>
<li><a href="#_编码一个神经元">6.3. 编码一个神经元</a></li>
<li><a href="#_把神经元组装成网络">6.4. 把神经元组装成网络</a></li>
<li><a href="#_例子前馈">6.5. 例子：前馈</a></li>
<li><a href="#_编码神经网络前馈">6.6. 编码神经网络：前馈</a></li>
<li><a href="#_训练神经网络第1部分">6.7. 训练神经网络，第1部分</a></li>
<li><a href="#_损失">6.8. 损失</a></li>
<li><a href="#_损失计算例子">6.9. 损失计算例子</a></li>
<li><a href="#_代码mse损失">6.10. 代码：MSE损失</a></li>
<li><a href="#_训练神经网络第2部分">6.11. 训练神经网络，第2部分</a></li>
<li><a href="#_例子计算偏导数">6.12. 例子：计算偏导数</a></li>
<li><a href="#_训练随机梯度下降">6.13. 训练：随机梯度下降</a></li>
<li><a href="#_代码一个完整的神经网络">6.14. 代码：一个完整的神经网络</a></li>
<li><a href="#_接下来">6.15. 接下来？</a></li>
<li><a href="#_tensorflow版本">6.16. TensorFlow版本</a></li>
</ul>
</li>
<li><a href="#_神经网络bp反向传播算法原理和详细推导流程">7. 神经网络BP反向传播算法原理和详细推导流程</a>
<ul class="sectlevel2">
<li><a href="#_反向传播算法和bp网络简介">7.1. 反向传播算法和BP网络简介</a></li>
<li><a href="#_信息前向传播">7.2. 信息前向传播</a></li>
<li><a href="#_误差反向传播">7.3. 误差反向传播</a>
<ul class="sectlevel3">
<li><a href="#_输出层的权重参数更新">7.3.1. 输出层的权重参数更新</a></li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
</div>
<div id="content">
<div id="preamble">
<div class="sectionbody">
<div class="imageblock">
<div class="content">
<img src="images/atguigu.jpg" alt="atguigu" width="300" height="200">
</div>
</div>
</div>
</div>
<div class="sect1">
<h2 id="_机器学习数学基础">1. 机器学习数学基础</h2>
<div class="sectionbody">
<div class="ulist">
<ul>
<li>
<p>高等数学</p>
</li>
<li>
<p>线性代数</p>
</li>
<li>
<p>概率论和统计学</p>
</li>
</ul>
</div>
<div class="sect2">
<h3 id="_线性代数">1.1. 线性代数</h3>
<div class="sect3">
<h4 id="_什么是矩阵">1.1.1. 什么是矩阵</h4>
<div class="ulist">
<ul>
<li>
<p>矩阵（Matrix）是一个按照长方阵列排列的复数或实数集合</p>
</li>
<li>
<p>矩阵最早来自于方程组的系数及常数所构成的方阵，最初是用来解决线性方程求解的工具</p>
</li>
<li>
<p>矩阵是高等代数中的常见工具，也常见于统计分析等应用数学学科中；矩阵在物理学和计算机科学中都有应用</p>
</li>
<li>
<p>矩阵的运算是数值分析领域的重要问题</p>
</li>
</ul>
</div>
</div>
<div class="sect3">
<h4 id="_矩阵">1.1.2. 矩阵</h4>
<div class="sect4">
<h5 id="_矩阵的定义">矩阵的定义</h5>
<div class="stemblock">
<div class="content">
\[A =
\begin{bmatrix}
    a_{11} &amp; a_{12} &amp; a_{13} &amp; \dots  &amp; a_{1n} \\
    a_{21} &amp; a_{22} &amp; a_{23} &amp; \dots  &amp; a_{2n} \\
    \vdots &amp; \vdots &amp; \vdots &amp; \ddots &amp; \vdots \\
    a_{m1} &amp; a_{m2} &amp; a_{m3} &amp; \dots  &amp; a_{mn}
\end{bmatrix}\]
</div>
</div>
<div class="paragraph">
<p>以上是\(m \times n\)矩阵的定义。</p>
</div>
<div class="ulist">
<ul>
<li>
<p>由\(m \times n\)个数\(a_{ij}, (i = 1,2,...,m;  j = 1,2,...,n)\)排成的m行n列的数表 A就称为m行n列的矩阵。</p>
</li>
<li>
<p>这\(m \times n\)个数称作矩阵A的元素\(a_{ij}\)，元素位于矩阵A的第i行第j列。</p>
</li>
<li>
<p>\(m \times n\)矩阵A可以记作\(A_{m \times n}\)，其中m是行数，n是列数，m, n &gt; 0。</p>
</li>
</ul>
</div>
</div>
<div class="sect4">
<h5 id="_特殊矩阵">特殊矩阵</h5>
<div class="paragraph">
<p>对于\(A_{m \times n}\)，如果m=n，即矩阵的行数与列数相等，那么称A为方阵。例如：</p>
</div>
<div class="stemblock">
<div class="content">
\[\begin{bmatrix}
    1 &amp; 2 &amp; 3 \\
    4 &amp; 5 &amp; 6 \\
    7 &amp; 8 &amp; 9
\end{bmatrix}\]
</div>
</div>
</div>
</div>
<div class="sect3">
<h4 id="_矩阵中的概念">1.1.3. 矩阵中的概念</h4>
<div class="ulist">
<ul>
<li>
<p>行数与列数都等于n的矩阵称为n阶矩阵，又称做n阶方阵，可以记作\(A_n\)。</p>
</li>
<li>
<p>只有一行的矩阵\(A_{1 \times n}\)称为行矩阵，又叫行向量。</p>
</li>
<li>
<p>同样，只有一列的矩阵\(A_{n \times 1}\)称为列矩阵，又叫列向量。</p>
</li>
<li>
<p>对于方阵，从左上角到右下角的直线，叫做主对角线，主对角线上的元素称为主对角线元素。</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[\begin{bmatrix}
    \textbf{1} &amp; 2 &amp; 3 \\
    4 &amp; \textbf{5} &amp; 6 \\
    7 &amp; 8 &amp; \textbf{9}
\end{bmatrix}\]
</div>
</div>
<div class="sect4">
<h5 id="_特殊矩阵_2">特殊矩阵</h5>
<div class="ulist">
<ul>
<li>
<p>矩阵的元素全部为0，称为零矩阵，用O表示。</p>
</li>
<li>
<p>对于方阵，如果只有对角线元素为1，其余元素都为0，那么称为单位矩阵，一般用I或者E表示。</p>
</li>
<li>
<p>对于方阵，不在对角线上的元素都为0，称为对角矩阵。</p>
</li>
</ul>
</div>
</div>
</div>
<div class="sect3">
<h4 id="_矩阵的加法">1.1.4. 矩阵的加法</h4>
<div class="ulist">
<ul>
<li>
<p>把矩阵的对应位元素相加。</p>
</li>
<li>
<p>矩阵的形状必须一致，即必须是同型矩阵。</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[\begin{bmatrix}
1 &amp; 2 \\
3 &amp; 4
\end{bmatrix}
+
\begin{bmatrix}
1 &amp; 1 \\
1 &amp; 1
\end{bmatrix}
=
\begin{bmatrix}
1+1 &amp; 2+1 \\
3+1 &amp; 4+1
\end{bmatrix}
=
\begin{bmatrix}
2 &amp; 3 \\
4 &amp; 5
\end{bmatrix}\]
</div>
</div>
</div>
<div class="sect3">
<h4 id="_矩阵的乘法">1.1.5. 矩阵的乘法</h4>
<div class="sect4">
<h5 id="_数与矩阵相乘">数与矩阵相乘</h5>
<div class="paragraph">
<p>数值与矩阵每一个元素相乘。</p>
</div>
<div class="stemblock">
<div class="content">
\[\begin{bmatrix}
1 &amp; 2 \\
3 &amp; 4
\end{bmatrix}
\times
2
=
\begin{bmatrix}
1 \times 2 &amp; 2 \times 2 \\
3 \times 2 &amp; 4 \times 2
\end{bmatrix}
=
\begin{bmatrix}
2 &amp; 4 \\
6 &amp; 8
\end{bmatrix}\]
</div>
</div>
</div>
<div class="sect4">
<h5 id="_矩阵与矩阵相乘">矩阵与矩阵相乘</h5>
<div class="ulist">
<ul>
<li>
<p>左矩阵的每一行与右矩阵的每一列，对应每一个元素相乘。</p>
</li>
<li>
<p>\(A \times B\)，那么有A矩阵\(m \times n\)，B 矩阵\(n \times k\)，要求左侧矩阵的列数n，必须等于右侧矩阵的行数n，结果矩阵C为\(m \times k\)矩阵。</p>
</li>
</ul>
</div>
<div class="paragraph">
<p>定义：</p>
</div>
<div class="stemblock">
<div class="content">
\[\begin{bmatrix}
    a_{11} &amp; a_{12} &amp; a_{13} \\
    a_{21} &amp; a_{22} &amp; a_{23}
\end{bmatrix}
\times
\begin{bmatrix}
    b_{11} &amp; b_{12} \\
    b_{21} &amp; b_{22} \\
    b_{31} &amp; b_{32}
\end{bmatrix}
=
\begin{bmatrix}
    a_{11}b_{11}+a_{12}b_{21}+a_{13}b_{31} &amp; a_{11}b_{12}+a_{12}b_{22}+a_{13}b_{32} \\
    a_{21}b_{11}+a_{22}b_{21}+a_{23}b_{31} &amp; a_{21}b_{12}+a_{22}b_{22}+a_{23}b_{32}
\end{bmatrix}\]
</div>
</div>
<div class="paragraph">
<p>练习：</p>
</div>
<div class="stemblock">
<div class="content">
\[\begin{bmatrix}
    1 &amp; 2 &amp; 3 \\
    4 &amp; 5 &amp; 6 \\
    7 &amp; 8 &amp; 9
\end{bmatrix}
\times
\begin{bmatrix}
    1 &amp; 2 &amp; 3 \\
    4 &amp; 5 &amp; 6 \\
    7 &amp; 8 &amp; 9
\end{bmatrix}
=?\]
</div>
</div>
</div>
</div>
<div class="sect3">
<h4 id="_矩阵的转置">1.1.6. 矩阵的转置</h4>
<div class="ulist">
<ul>
<li>
<p>把矩阵A的行换成相同序数的列，得到一个新矩阵，叫做A的转置矩阵，记作\(A^T\)</p>
</li>
<li>
<p>行变列，列变行</p>
</li>
<li>
<p>A 为\(m \times n\)矩阵，转置之后为\(n \times m\)矩阵</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[\begin{bmatrix}
    a_{11} &amp; a_{12} &amp; a_{13} \\
    a_{21} &amp; a_{22} &amp; a_{23}
\end{bmatrix}
\rightarrow
\begin{bmatrix}
    a_{11} &amp; a_{21} \\
    a_{12} &amp; a_{22} \\
    a_{13} &amp; a_{23}
\end{bmatrix}\]
</div>
</div>
</div>
<div class="sect3">
<h4 id="_矩阵的运算方法">1.1.7. 矩阵的运算方法</h4>
<div class="sect4">
<h5 id="_加法">加法</h5>
<div class="ulist">
<ul>
<li>
<p>\(A + B = B + A\)</p>
</li>
<li>
<p>\((A + B) + C = A + (B + C)\)</p>
</li>
</ul>
</div>
</div>
<div class="sect4">
<h5 id="_乘法">乘法</h5>
<div class="ulist">
<ul>
<li>
<p>\((λμ)A = λ(μA)\)</p>
</li>
<li>
<p>\((λ + μ)A = λA + μA\)</p>
</li>
<li>
<p>\(λ(A + B) = λA + λB\)</p>
</li>
<li>
<p>\((AB) C = A(BC)\)</p>
</li>
<li>
<p>\(λ(AB) = (λA)B = A(λB)\)</p>
</li>
<li>
<p>\(A(B + C) = AB + AC\)</p>
</li>
<li>
<p>\((B + C)A = BA + CA\)</p>
</li>
</ul>
</div>
</div>
<div class="sect4">
<h5 id="_减法">减法</h5>
<div class="ulist">
<ul>
<li>
<p>\(A - B = A + B × (-1)\)</p>
</li>
<li>
<p>\(A - A = A + (-A) = O\)</p>
</li>
</ul>
</div>
</div>
<div class="sect4">
<h5 id="_转置">转置</h5>
<div class="ulist">
<ul>
<li>
<p>\((A^T)^T = A\)</p>
</li>
<li>
<p>\((A + B)^T = A^T + B^T\)</p>
</li>
<li>
<p>\((λA)^T = λA^T\)</p>
</li>
<li>
<p>\((AB)^T = B^T A^T\)</p>
</li>
</ul>
</div>
</div>
</div>
<div class="sect3">
<h4 id="_矩阵的逆">1.1.8. 矩阵的逆</h4>
<div class="ulist">
<ul>
<li>
<p>对于n阶方阵A，如果有一个n阶方阵B，使得\(AB = BA = E\)，就称矩阵A是可逆的，并把B称为A的逆矩阵。</p>
</li>
<li>
<p>A的逆矩阵记作\(A^{-1}\)，如果\(AB = BA = E\)，则\(B = A-1\)。</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[\begin{bmatrix}
  1 &amp; 2 &amp; 3 \\
  2 &amp; 2 &amp; 1 \\
  3 &amp; 4 &amp; 3
\end{bmatrix}
\times
\begin{bmatrix}
  1 &amp; 3 &amp; -2 \\
  -\frac{3}{2} &amp; -3 &amp; \frac{5}{2} \\
  1 &amp; 1 &amp; -1
\end{bmatrix}
=
\begin{bmatrix}
1 &amp; 0 &amp; 0 \\
0 &amp; 1 &amp; 0 \\
0 &amp; 0 &amp; 1
\end{bmatrix}\]
</div>
</div>
</div>
</div>
<div class="sect2">
<h3 id="_微积分基础知识">1.2. 微积分基础知识</h3>
<div class="ulist">
<ul>
<li>
<p>什么是导数</p>
</li>
<li>
<p>偏导数</p>
</li>
<li>
<p>方向导数和梯度</p>
</li>
<li>
<p>凸函数和凹函数</p>
</li>
</ul>
</div>
<div class="sect3">
<h4 id="_什么是导数">1.2.1. 什么是导数</h4>
<div class="imageblock">
<div class="content">
<img src="images/derivative.png" alt="derivative">
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>导数反映的是函数y = f(x)在某一点处沿x轴正方向的变化率</p>
</li>
<li>
<p>在x轴上某一点处，如果f'(x)&gt;0，说明f(x)的函数值在x点沿x轴正方向是趋于增加的；如果f'(x)&lt;0，说明f(x)的函数值在x点沿x轴正方向是趋于减少的</p>
</li>
</ul>
</div>
</div>
<div class="sect3">
<h4 id="_偏导数">1.2.2. 偏导数</h4>
<div class="imageblock">
<div class="content">
<img src="images/piandaoshu.png" alt="piandaoshu">
</div>
</div>
<div class="stemblock">
<div class="content">
\[\frac{\partial}{\partial x_j} f(x_0,x_1,\dots,x_n)
=
\lim_{\Delta x \to 0} \frac{\Delta y}{\Delta x}
=
\lim_{\Delta x \to 0} \frac{f(x_0,\dots,x_j+\Delta x, \dots, x_n)-f(x_0,\dots,x_j,\dots,x_n)}{\Delta x}\]
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>导数与偏导数本质是一致的，都是当自变量的变化量趋于0时，函数值的变化量与自变量变化量比值的极限</p>
</li>
<li>
<p>偏导数也就是函数在某一点上沿某个坐标轴正方向的的变化率</p>
</li>
<li>
<p>导数指的是一元函数中，函数y=f(x)在某一点处沿x轴正方向的变化率；而偏导数，指的是多元函数中，函数\(y=f(x_1,x_2,…,x_n)\)在某一点处沿某一坐标轴\((x_1,x_2,…,x_n)\)正方向的变化率</p>
</li>
</ul>
</div>
</div>
<div class="sect3">
<h4 id="_方向导数">1.2.3. 方向导数</h4>
<div class="ulist">
<ul>
<li>
<p>函数某一点在某一趋近方向（向量方向）上的导数值</p>
</li>
<li>
<p>方向导数就是函数在除坐标轴正方向外，其他特定方向上的变化率</p>
</li>
</ul>
</div>
<div class="imageblock">
<div class="content">
<img src="images/fangxiang.png" alt="fangxiang">
</div>
</div>
<div class="stemblock">
<div class="content">
\[\frac{\partial}{\partial l} f(x_0, x_1, \dots, x_n)
=
\lim_{\rho \to 0} \frac{\Delta y}{\Delta x}
=
\lim_{\rho \to 0} \frac{f(x_0+\Delta x_0,\dots,x_j+\Delta x_j, \dots, x_n+\Delta x_n)-f(x_0,\dots,x_1,\dots,x_n)}{\rho} \\
其中，\rho=\sqrt{(\Delta x_0)^2+\dots+(\Delta x_j)^2+(\Delta x_n)^2} \\
l = (\Delta x_0, \Delta x_1, \dots, \Delta x_n)\]
</div>
</div>
</div>
<div class="sect3">
<h4 id="_梯度gradient">1.2.4. 梯度（Gradient）</h4>
<div class="paragraph">
<p>问题：函数在变量空间的某一点处，沿着哪一个方向有最大的变化率？</p>
</div>
<div class="paragraph">
<p>答案：梯度。</p>
</div>
<div class="stemblock">
<div class="content">
\[gradf(x_0, x_1, \dots, x_n)=(\frac{\partial f}{\partial x_0}, \frac{\partial f}{\partial x_1}, \dots, \frac{\partial f}{\partial x_n})\]
</div>
</div>
<div class="imageblock">
<div class="content">
<img src="images/tidu.png" alt="tidu">
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>定义：函数在某一点的梯度是这样一个向量，它的方向与取得最大方向导数的方向一致，而它的模为方向导数的最大值</p>
</li>
<li>
<p>梯度是一个向量，即有方向、有大小； </p>
</li>
<li>
<p>梯度的方向是最大方向导数的方向；梯度的值是最大方向导数的值</p>
</li>
</ul>
</div>
</div>
<div class="sect3">
<h4 id="_凸函数和凹函数">1.2.5. 凸函数和凹函数</h4>
<div class="paragraph">
<p><span class="image"><img src="images/tuao.png" alt="tuao"></span></p>
</div>
<div class="sect4">
<h5 id="_凸函数">凸函数</h5>
<div class="paragraph">
<p>凸函数是具有如下特性的一个定义在某个向量空间的凸子集C（区间）上的实值函数f：对其定义域C上的任意两点\(x_{1},x_{2}\)，总有\(f(\frac{x_1+x_2}{2}) \le \frac{f(x_1)+f(x_2)}{2}\)。</p>
</div>
<div class="imageblock">
<div class="content">
<img src="images/tuhanshu.png" alt="tuhanshu">
</div>
</div>
</div>
<div class="sect4">
<h5 id="_凹函数">凹函数</h5>
<div class="paragraph">
<p>我们称一个有实值函数f在某区间（或者某个向量空间中的凹集）上是凹的，如果对任意该区间内不相等的x和y和[0,1]中的任意t有</p>
</div>
<div class="stemblock">
<div class="content">
\[f(tx+(1-t)y) \ge tf(x)+(1-t)f(y)\]
</div>
</div>
<div class="paragraph">
<p>某函数f:R→R，在x和y之间的每一点z，在图中的点(z, f(z))是在以点(x, f(x)) and (y, f(y))连成的直线之上。</p>
</div>
<div class="imageblock">
<div class="content">
<img src="images/aohanshu.png" alt="aohanshu">
</div>
</div>
</div>
</div>
</div>
<div class="sect2">
<h3 id="_概率论和统计学">1.3. 概率论和统计学</h3>
<div class="ulist">
<ul>
<li>
<p>常用统计变量</p>
</li>
<li>
<p>常见概率分布</p>
</li>
<li>
<p>重要概率公式</p>
</li>
</ul>
</div>
<div class="sect3">
<h4 id="_常用统计变量">1.3.1. 常用统计变量</h4>
<div class="ulist">
<ul>
<li>
<p>样本均值</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[E(X)=\bar{X}=\frac{1}{n}\sum_{i=1}^n X_i\]
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>样本方差</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[D(X)=S^2=\frac{1}{n-1}\sum_{i=1}^n (X_i-\bar{X})^2
=\frac{1}{n-1}\sum_{i=1}^n (X^2_i-n \bar{X})\]
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>样本标准差</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[\sqrt{D(X)}=S=\sqrt{\frac{1}{n-1}\sum_{i=1}^n (X_i-\bar{X})^2}\]
</div>
</div>
</div>
<div class="sect3">
<h4 id="_常见概率分布">1.3.2. 常见概率分布</h4>
<div class="ulist">
<ul>
<li>
<p>均匀分布</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[f(x)=\frac{1}{b-a}, a &lt; x &lt; b \\
f(x)=0, else\]
</div>
</div>
<div class="imageblock">
<div class="content">
<img src="images/junyunfenbu.png" alt="junyunfenbu">
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>正态分布（高斯分布）</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[f(x)=\frac{1}{\sqrt {2 \pi} \sigma}exp(-\frac{(x-\mu)^2}{2\sigma^2})\]
</div>
</div>
<div class="imageblock">
<div class="content">
<img src="images/zhengtai.png" alt="zhengtai">
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>指数分布</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[\begin{equation}
f(x)
=
\begin{cases}
    \lambda e^{- \lambda x} &amp; x \ge 0 \\
    0 &amp; x &lt; 0
\end{cases}
\end{equation}\]
</div>
</div>
<div class="imageblock">
<div class="content">
<img src="images/zhishu.png" alt="zhishu">
</div>
</div>
</div>
<div class="sect3">
<h4 id="_重要概率公式">1.3.3. 重要概率公式</h4>
<div class="ulist">
<ul>
<li>
<p>条件概率公式</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[P(B \vert A)=\frac{P(AB)}{P(A)}\]
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>全概率公式</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[P(A) = P(A \vert B_1)P(B_1) + P(A \vert B_2)P(B_2) + \dots + P(A \vert B_n)P(B_n)\]
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>贝叶斯公式</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[P(B_i \vert A) = \frac{P(A \vert B_i)P(B_i)}{\sum_{j=1}^{n}P(A \vert B_j)P(B_j)}\]
</div>
</div>
</div>
</div>
</div>
</div>
<div class="sect1">
<h2 id="_数学基础精简进阶版">2. 数学基础(精简进阶版)</h2>
<div class="sectionbody">
<div class="sect2">
<h3 id="_线性代数_2">2.1. 线性代数</h3>
<div class="paragraph">
<p>下面分别概括了向量、矩阵、运算、范数、特征向量和特征值的概念。</p>
</div>
<div class="sect3">
<h4 id="_向量">2.1.1. 向量</h4>
<div class="paragraph">
<p>本书中的向量指的是列向量。一个\(n\)维向量\(\boldsymbol{x}\)的表达式可写成</p>
</div>
<div class="stemblock">
<div class="content">
\[\boldsymbol{x} =
\begin{bmatrix}
    x_{1}  \\
    x_{2}  \\
    \vdots  \\
    x_{n}
\end{bmatrix},\]
</div>
</div>
<div class="paragraph">
<p>其中\(x_1, \ldots, x_n\)是向量的元素。我们将各元素均为实数的\(n\)维向量\(\boldsymbol{x}\)记作\(\boldsymbol{x} \in \mathbb{R}^{n}\)或\(\boldsymbol{x} \in \mathbb{R}^{n \times 1}\)。</p>
</div>
</div>
<div class="sect3">
<h4 id="_矩阵_2">2.1.2. 矩阵</h4>
<div class="paragraph">
<p>一个\(m\)行\(n\)列矩阵的表达式可写成</p>
</div>
<div class="stemblock">
<div class="content">
\[\boldsymbol{X} =
\begin{bmatrix}
    x_{11} &amp; x_{12}  &amp; \dots  &amp; x_{1n} \\
    x_{21} &amp; x_{22}  &amp; \dots  &amp; x_{2n} \\
    \vdots &amp; \vdots  &amp; \ddots &amp; \vdots \\
    x_{m1} &amp; x_{m2}  &amp; \dots  &amp; x_{mn}
\end{bmatrix},\]
</div>
</div>
<div class="paragraph">
<p>其中\(x_{ij}\)是矩阵\(\boldsymbol{X}\)中第\(i\)行第\(j\)列的元素（\(1 \leq i \leq m, 1 \leq j \leq n\)）。我们将各元素均为实数的\(m\)行\(n\)列矩阵\(\boldsymbol{X}\)记作\(\boldsymbol{X} \in \mathbb{R}^{m \times n}\)。不难发现，向量是特殊的矩阵。</p>
</div>
</div>
<div class="sect3">
<h4 id="_运算">2.1.3. 运算</h4>
<div class="paragraph">
<p>设\(n\)维向量\(\boldsymbol{A}\)中的元素为\(a_1, \ldots, a_n\)，\(n\)维向量\(\boldsymbol{b}\)中的元素为\(b_1, \ldots, b_n\)。向量\(\boldsymbol{A}\)与\(\boldsymbol{b}\)的点乘（内积）是一个标量：</p>
</div>
<div class="stemblock">
<div class="content">
\[\boldsymbol{a} \cdot \boldsymbol{b} = a_1 b_1 + \ldots + a_n b_n.\]
</div>
</div>
<div class="paragraph">
<p>设两个\(m\)行\(n\)列矩阵</p>
</div>
<div class="stemblock">
<div class="content">
\[\boldsymbol{A} =
\begin{bmatrix}
    a_{11} &amp; a_{12} &amp; \dots  &amp; a_{1n} \\
    a_{21} &amp; a_{22} &amp; \dots  &amp; a_{2n} \\
    \vdots &amp; \vdots &amp; \ddots &amp; \vdots \\
    a_{m1} &amp; a_{m2} &amp; \dots  &amp; a_{mn}
\end{bmatrix},\quad
\boldsymbol{B} =
\begin{bmatrix}
    b_{11} &amp; b_{12} &amp; \dots  &amp; b_{1n} \\
    b_{21} &amp; b_{22} &amp; \dots  &amp; b_{2n} \\
    \vdots &amp; \vdots &amp; \ddots &amp; \vdots \\
    b_{m1} &amp; b_{m2} &amp; \dots  &amp; b_{mn}
\end{bmatrix}.\]
</div>
</div>
<div class="paragraph">
<p>矩阵\(\boldsymbol{A}\)的转置是一个\(n\)行\(m\)列矩阵，它的每一行其实是原矩阵的每一列：</p>
</div>
<div class="stemblock">
<div class="content">
\[\boldsymbol{A}^\top =
\begin{bmatrix}
    a_{11} &amp; a_{21} &amp; \dots  &amp; a_{m1} \\
    a_{12} &amp; a_{22} &amp; \dots  &amp; a_{m2} \\
    \vdots &amp; \vdots &amp; \ddots &amp; \vdots \\
    a_{1n} &amp; a_{2n} &amp; \dots  &amp; a_{mn}
\end{bmatrix}.\]
</div>
</div>
<div class="paragraph">
<p>两个相同形状的矩阵的加法是将两个矩阵按元素做加法：</p>
</div>
<div class="stemblock">
<div class="content">
\[\boldsymbol{A} + \boldsymbol{B} =
\begin{bmatrix}
    a_{11} + b_{11} &amp; a_{12} + b_{12} &amp; \dots  &amp; a_{1n} + b_{1n} \\
    a_{21} + b_{21} &amp; a_{22} + b_{22} &amp; \dots  &amp; a_{2n} + b_{2n} \\
    \vdots &amp; \vdots &amp; \ddots &amp; \vdots \\
    a_{m1} + b_{m1} &amp; a_{m2} + b_{m2} &amp; \dots  &amp; a_{mn} + b_{mn}
\end{bmatrix}.\]
</div>
</div>
<div class="paragraph">
<p>我们使用符号\(\odot\)表示两个矩阵按元素做乘法的运算：</p>
</div>
<div class="stemblock">
<div class="content">
\[\boldsymbol{A} \odot \boldsymbol{B} =
\begin{bmatrix}
    a_{11}  b_{11} &amp; a_{12}  b_{12} &amp; \dots  &amp; a_{1n}  b_{1n} \\
    a_{21}  b_{21} &amp; a_{22}  b_{22} &amp; \dots  &amp; a_{2n}  b_{2n} \\
    \vdots &amp; \vdots &amp; \ddots &amp; \vdots \\
    a_{m1}  b_{m1} &amp; a_{m2}  b_{m2} &amp; \dots  &amp; a_{mn}  b_{mn}
\end{bmatrix}.\]
</div>
</div>
<div class="paragraph">
<p>定义一个标量\(k\)。标量与矩阵的乘法也是按元素做乘法的运算：</p>
</div>
<div class="stemblock">
<div class="content">
\[k\boldsymbol{A} =
\begin{bmatrix}
    ka_{11} &amp; ka_{12} &amp; \dots  &amp; ka_{1n} \\
    ka_{21} &amp; ka_{22} &amp; \dots  &amp; ka_{2n} \\
    \vdots &amp; \vdots   &amp; \ddots &amp; \vdots \\
    ka_{m1} &amp; ka_{m2} &amp; \dots  &amp; ka_{mn}
\end{bmatrix}.\]
</div>
</div>
<div class="paragraph">
<p>其他诸如标量与矩阵按元素相加、相除等运算与上式中的相乘运算类似。矩阵按元素开根号、取对数等运算也就是对矩阵每个元素开根号、取对数等，并得到和原矩阵形状相同的矩阵。</p>
</div>
<div class="paragraph">
<p>矩阵乘法和按元素的乘法不同。设\(\boldsymbol{A}\)为\(m\)行\(p\)列的矩阵，\(\boldsymbol{B}\)为\(p\)行\(n\)列的矩阵。两个矩阵相乘的结果</p>
</div>
<div class="stemblock">
<div class="content">
\[\boldsymbol{A} \boldsymbol{B} =
\begin{bmatrix}
    a_{11} &amp; a_{12} &amp; \dots  &amp; a_{1p} \\
    a_{21} &amp; a_{22} &amp; \dots  &amp; a_{2p} \\
    \vdots &amp; \vdots &amp; \ddots &amp; \vdots \\
    a_{i1} &amp; a_{i2} &amp; \dots  &amp; a_{ip} \\
    \vdots &amp; \vdots &amp; \ddots &amp; \vdots \\
    a_{m1} &amp; a_{m2} &amp; \dots  &amp; a_{mp}
\end{bmatrix}
\begin{bmatrix}
    b_{11} &amp; b_{12} &amp; \dots  &amp; b_{1j} &amp; \dots &amp; b_{1n} \\
    b_{21} &amp; b_{22} &amp; \dots  &amp; b_{2j} &amp; \dots  &amp; b_{2n} \\
    \vdots &amp; \vdots &amp; \ddots &amp; \vdots &amp; \ddots &amp; \vdots \\
    b_{p1} &amp; b_{p2} &amp; \dots  &amp; b_{pj} &amp; \dots  &amp; b_{pn}
\end{bmatrix}\]
</div>
</div>
<div class="paragraph">
<p>是一个\(m\)行\(n\)列的矩阵，其中第\(i\)行第\(j\)列（\(1 \leq i \leq m, 1 \leq j \leq n\)）的元素为</p>
</div>
<div class="stemblock">
<div class="content">
\[a_{i1}b_{1j}  + a_{i2}b_{2j} + \ldots + a_{ip}b_{pj} = \sum_{k=1}^p a_{ik}b_{kj}.\]
</div>
</div>
</div>
<div class="sect3">
<h4 id="_范数">2.1.4. 范数</h4>
<div class="paragraph">
<p>设\(n\)维向量\(\boldsymbol{x}\)中的元素为\(x_1, \ldots, x_n\)。向量\(\boldsymbol{x}\)的\(L_p\)范数为</p>
</div>
<div class="stemblock">
<div class="content">
\[\|\boldsymbol{x}\|_p = \left(\sum_{i=1}^n \left|x_i \right|^p \right)^{1/p}.\]
</div>
</div>
<div class="paragraph">
<p>例如，\(\boldsymbol{x}\)的\(L_1\)范数是该向量元素绝对值之和：</p>
</div>
<div class="stemblock">
<div class="content">
\[\|\boldsymbol{x}\|_1 = \sum_{i=1}^n \left|x_i \right|.\]
</div>
</div>
<div class="paragraph">
<p>而\(\boldsymbol{x}\)的\(L_2\)范数是该向量元素平方和的平方根：</p>
</div>
<div class="stemblock">
<div class="content">
\[\|\boldsymbol{x}\|_2 = \sqrt{\sum_{i=1}^n x_i^2}.\]
</div>
</div>
<div class="paragraph">
<p>我们通常用\(\|\boldsymbol{x}\|\)指代\(\|\boldsymbol{x}\|_2\)。</p>
</div>
<div class="paragraph">
<p>设\(\boldsymbol{x}\)是一个\(m\)行\(n\)列矩阵。矩阵\(\boldsymbol{x}\)的Frobenius范数为该矩阵元素平方和的平方根：</p>
</div>
<div class="stemblock">
<div class="content">
\[\|\boldsymbol{X}\|_F = \sqrt{\sum_{i=1}^m \sum_{j=1}^n x_{ij}^2},\]
</div>
</div>
<div class="paragraph">
<p>其中\(x_{ij}\)为矩阵\(\boldsymbol{x}\)在第\(i\)行第\(j\)列的元素。</p>
</div>
</div>
<div class="sect3">
<h4 id="_特征向量和特征值">2.1.5. 特征向量和特征值</h4>
<div class="paragraph">
<p>对于一个\(n\)行\(n\)列的矩阵\(\boldsymbol{A}\)，假设有标量\(\lambda\)和非零的\(n\)维向量\(\boldsymbol{v}\)使</p>
</div>
<div class="stemblock">
<div class="content">
\[\boldsymbol{A} \boldsymbol{v} = \lambda \boldsymbol{v},\]
</div>
</div>
<div class="paragraph">
<p>那么\(\boldsymbol{v}\)是矩阵\(\boldsymbol{A}\)的一个特征向量，标量\(\lambda\)是\(\boldsymbol{v}\)对应的特征值。</p>
</div>
</div>
</div>
<div class="sect2">
<h3 id="_微分">2.2. 微分</h3>
<div class="paragraph">
<p>我们在这里简要介绍微分的一些基本概念和演算。</p>
</div>
<div class="sect3">
<h4 id="_导数和微分">2.2.1. 导数和微分</h4>
<div class="paragraph">
<p>假设函数\(f: \mathbb{R} \rightarrow \mathbb{R}\)的输入和输出都是标量。函数\(f\)的导数</p>
</div>
<div class="stemblock">
<div class="content">
\[f'(x) = \lim_{h \rightarrow 0} \frac{f(x+h) - f(x)}{h},\]
</div>
</div>
<div class="paragraph">
<p>且假定该极限存在。给定\(y = f(x)\)，其中\(x\)和\(y\)分别是函数\(f\)的自变量和因变量。以下有关导数和微分的表达式等价：</p>
</div>
<div class="stemblock">
<div class="content">
\[f'(x) = y' = \frac{\text{d}y}{\text{d}x} = \frac{\text{d}f}{\text{d}x} = \frac{\text{d}}{\text{d}x} f(x) = \text{D}f(x) = \text{D}_x f(x),\]
</div>
</div>
<div class="paragraph">
<p>其中符号\(\text{D}\)和\(\text{d}/\text{d}x\)也叫微分运算符。常见的微分演算有\(\text{D}C = 0\)（\(C\)为常数）、\(\text{D}x^n = nx^{n-1}\)（\(n\)为常数）、\(\text{D}e^x = e^x\)、\(\text{D}\ln(x) = 1/x\)等。</p>
</div>
<div class="paragraph">
<p>如果函数\(f\)和\(g\)都可导，设\(C\)为常数，那么</p>
</div>
<div class="stemblock">
<div class="content">
\[\begin{aligned}
\frac{\text{d}}{\text{d}x} [Cf(x)] &amp;= C \frac{\text{d}}{\text{d}x} f(x),\\
\frac{\text{d}}{\text{d}x} [f(x) + g(x)] &amp;= \frac{\text{d}}{\text{d}x} f(x) + \frac{\text{d}}{\text{d}x} g(x),\\
\frac{\text{d}}{\text{d}x} [f(x)g(x)] &amp;= f(x) \frac{\text{d}}{\text{d}x} [g(x)] + g(x) \frac{\text{d}}{\text{d}x} [f(x)],\\
\frac{\text{d}}{\text{d}x} \left[\frac{f(x)}{g(x)}\right] &amp;= \frac{g(x) \frac{\text{d}}{\text{d}x} [f(x)] - f(x) \frac{\text{d}}{\text{d}x} [g(x)]}{[g(x)]^2}.
\end{aligned}\]
</div>
</div>
<div class="paragraph">
<p>如果\(y=f(u)\)和\(u=g(x)\)都是可导函数，依据链式法则，</p>
</div>
<div class="stemblock">
<div class="content">
\[\frac{\text{d}y}{\text{d}x} = \frac{\text{d}y}{\text{d}u} \frac{\text{d}u}{\text{d}x}.\]
</div>
</div>
</div>
<div class="sect3">
<h4 id="_泰勒展开">2.2.2. 泰勒展开</h4>
<div class="paragraph">
<p>函数\(f\)的泰勒展开式是</p>
</div>
<div class="stemblock">
<div class="content">
\[f(x) = \sum_{n=0}^\infty \frac{f^{(n)}(a)}{n!} (x-a)^n,\]
</div>
</div>
<div class="paragraph">
<p>其中\(f^{(n)}\)为函数\(f\)的\(n\)阶导数（求\(n\)次导数），\(n!\)为\(n\)的阶乘。假设\(\epsilon\)是一个足够小的数，如果将上式中\(x\)和\(a\)分别替换成\(x+\epsilon\)和\(x\)，可以得到</p>
</div>
<div class="stemblock">
<div class="content">
\[f(x + \epsilon) \approx f(x) + f'(x) \epsilon + \mathcal{O}(\epsilon^2).\]
</div>
</div>
<div class="paragraph">
<p>由于\(\epsilon\)足够小，上式也可以简化成</p>
</div>
<div class="stemblock">
<div class="content">
\[f(x + \epsilon) \approx f(x) + f'(x) \epsilon.\]
</div>
</div>
</div>
<div class="sect3">
<h4 id="_偏导数_2">2.2.3. 偏导数</h4>
<div class="paragraph">
<p>设\(u\)为一个有\(n\)个自变量的函数，\(u = f(x_1, x_2, \ldots, x_n)\)，它有关第\(i\)个变量\(x_i\)的偏导数为</p>
</div>
<div class="stemblock">
<div class="content">
\[\frac{\partial u}{\partial x_i} = \lim_{h \rightarrow 0} \frac{f(x_1, \ldots, x_{i-1}, x_i+h, x_{i+1}, \ldots, x_n) - f(x_1, \ldots, x_i, \ldots, x_n)}{h}.\]
</div>
</div>
<div class="paragraph">
<p>以下有关偏导数的表达式等价：</p>
</div>
<div class="stemblock">
<div class="content">
\[\frac{\partial u}{\partial x_i} = \frac{\partial f}{\partial x_i} = f_{x_i} = f_i = \text{D}_i f = \text{D}_{x_i} f.\]
</div>
</div>
<div class="paragraph">
<p>为了计算\(\partial u/\partial x_i\)，只需将\(x_1, \ldots, x_{i-1}, x_{i+1}, \ldots, x_n\)视为常数并求\(u\)有关\(x_i\)的导数。</p>
</div>
</div>
<div class="sect3">
<h4 id="_梯度">2.2.4. 梯度</h4>
<div class="paragraph">
<p>假设函数\(f: \mathbb{R}^n \rightarrow \mathbb{R}\)的输入是一个\(n\)维向量\(\boldsymbol{x} = [x_1, x_2, \ldots, x_n ]^\top\)，输出是标量。函数\(f(\boldsymbol{x})\)有关\(\boldsymbol{x}\)的梯度是一个由\(n\)个偏导数组成的向量：</p>
</div>
<div class="stemblock">
<div class="content">
\[\nabla_{\boldsymbol{x}} f(\boldsymbol{x}) = \bigg[\frac{\partial f(\boldsymbol{x})}{\partial x_1}, \frac{\partial f(\boldsymbol{x})}{\partial x_2}, \ldots, \frac{\partial f(\boldsymbol{x})}{\partial x_n}\bigg]^\top.\]
</div>
</div>
<div class="paragraph">
<p>为表示简洁，我们有时用\(\nabla f(\boldsymbol{x})\)代替\(\nabla_{\boldsymbol{x}} f(\boldsymbol{x})\)。</p>
</div>
<div class="paragraph">
<p>假设\(\boldsymbol{x}\)是一个向量，常见的梯度演算包括</p>
</div>
<div class="stemblock">
<div class="content">
\[\begin{aligned}
\nabla_{\boldsymbol{x}} \boldsymbol{A}^\top \boldsymbol{x} &amp;= \boldsymbol{A}, \\
\nabla_{\boldsymbol{x}} \boldsymbol{x}^\top \boldsymbol{A}  &amp;= \boldsymbol{A}, \\
\nabla_{\boldsymbol{x}} \boldsymbol{x}^\top \boldsymbol{A} \boldsymbol{x}  &amp;= (\boldsymbol{A} + \boldsymbol{A}^\top)\boldsymbol{x},\\
\nabla_{\boldsymbol{x}} \|\boldsymbol{x} \|^2 &amp;= \nabla_{\boldsymbol{x}} \boldsymbol{x}^\top \boldsymbol{x} = 2\boldsymbol{x}.
\end{aligned}\]
</div>
</div>
<div class="paragraph">
<p>类似地，假设\(\boldsymbol{x}\)是一个矩阵，那么</p>
</div>
<div class="stemblock">
<div class="content">
\[\nabla_{\boldsymbol{X}} \|\boldsymbol{X} \|_F^2 = 2\boldsymbol{X}.\]
</div>
</div>
</div>
<div class="sect3">
<h4 id="_海森矩阵">2.2.5. 海森矩阵</h4>
<div class="paragraph">
<p>假设函数\(f: \mathbb{R}^n \rightarrow \mathbb{R}\)的输入是一个\(n\)维向量\(\boldsymbol{x} = [x_1, x_2, \ldots, x_n ]^\top\)，输出是标量。假定函数\(f\)所有的二阶偏导数都存在，\(f\)的海森矩阵\(\boldsymbol{H}\)是一个\(n\)行\(n\)列的矩阵：</p>
</div>
<div class="stemblock">
<div class="content">
\[\boldsymbol{H} =
\begin{bmatrix}
    \frac{\partial^2 f}{\partial x_1^2} &amp; \frac{\partial^2 f}{\partial x_1 \partial x_2} &amp; \dots  &amp; \frac{\partial^2 f}{\partial x_1 \partial x_n} \\
    \frac{\partial^2 f}{\partial x_2 \partial x_1} &amp; \frac{\partial^2 f}{\partial x_2^2} &amp; \dots  &amp; \frac{\partial^2 f}{\partial x_2 \partial x_n} \\
    \vdots &amp; \vdots &amp; \ddots &amp; \vdots \\
    \frac{\partial^2 f}{\partial x_n \partial x_1} &amp; \frac{\partial^2 f}{\partial x_n \partial x_2} &amp; \dots  &amp; \frac{\partial^2 f}{\partial x_n^2}
\end{bmatrix},\]
</div>
</div>
<div class="paragraph">
<p>其中二阶偏导数</p>
</div>
<div class="stemblock">
<div class="content">
\[\frac{\partial^2 f}{\partial x_i \partial x_j} = \frac{\partial }{\partial x_j} \left(\frac{\partial f}{ \partial x_i}\right).\]
</div>
</div>
</div>
</div>
<div class="sect2">
<h3 id="_概率">2.3. 概率</h3>
<div class="paragraph">
<p>最后，我们简要介绍条件概率、期望和均匀分布。</p>
</div>
<div class="sect3">
<h4 id="_条件概率">2.3.1. 条件概率</h4>
<div class="paragraph">
<p>假设事件\(A\)和事件\(B\)的概率分别为\(P(A)\)和\(P(B)\)，两个事件同时发生的概率记作\(P(A \cap B)\)或\(P(A, B)\)。给定事件\(B\)，事件\(A\)的条件概率</p>
</div>
<div class="stemblock">
<div class="content">
\[P(A \mid B) = \frac{P(A \cap B)}{P(B)}.\]
</div>
</div>
<div class="paragraph">
<p>也就是说，</p>
</div>
<div class="stemblock">
<div class="content">
\[P(A \cap B) = P(B) P(A \mid B) = P(A) P(B \mid A).\]
</div>
</div>
<div class="paragraph">
<p>当满足</p>
</div>
<div class="stemblock">
<div class="content">
\[P(A \cap B) = P(A) P(B)\]
</div>
</div>
<div class="paragraph">
<p>时，事件\(A\)和事件\(B\)相互独立。</p>
</div>
</div>
<div class="sect3">
<h4 id="_期望">2.3.2. 期望</h4>
<div class="paragraph">
<p>离散的随机变量\(X\)的期望（或平均值）为</p>
</div>
<div class="stemblock">
<div class="content">
\[E(X) = \sum_{x} x P(X = x).\]
</div>
</div>
</div>
<div class="sect3">
<h4 id="_均匀分布">2.3.3. 均匀分布</h4>
<div class="paragraph">
<p>假设随机变量\(X\)服从\([a, b\)]上的均匀分布，即\(X \sim U(a, b)\)。随机变量\(X\)取\(a\)和\(b\)之间任意一个数的概率相等。</p>
</div>
</div>
</div>
</div>
</div>
<div class="sect1">
<h2 id="_机器学习简介">3. 机器学习简介</h2>
<div class="sectionbody">
<div class="ulist">
<ul>
<li>
<p>机器学习的概念</p>
</li>
<li>
<p>机器学习主要分类</p>
</li>
<li>
<p>监督学习三要素</p>
</li>
<li>
<p>监督学习模型评估策略</p>
</li>
<li>
<p>监督学习模型求解算法</p>
</li>
</ul>
</div>
<div class="sect2">
<h3 id="_机器学习的概念">3.1. 机器学习的概念</h3>
<div class="ulist">
<ul>
<li>
<p>机器学习是什么</p>
</li>
<li>
<p>机器学习的开端</p>
</li>
<li>
<p>机器学习的定义</p>
</li>
<li>
<p>机器学习的过程</p>
</li>
<li>
<p>机器学习示例</p>
</li>
</ul>
</div>
<div class="sect3">
<h4 id="_机器学习是什么">3.1.1. 机器学习是什么</h4>
<div class="ulist">
<ul>
<li>
<p>什么是学习</p>
<div class="ulist">
<ul>
<li>
<p>从人的学习说起</p>
</li>
<li>
<p>学习理论；从实践经验中总结</p>
</li>
<li>
<p>在理论上推导；在实践中检验</p>
</li>
<li>
<p>通过各种手段获取知识或技能的过程</p>
</li>
</ul>
</div>
</li>
<li>
<p>机器怎么学习？</p>
<div class="ulist">
<ul>
<li>
<p>处理某个特定的任务，以大量的“经验”为基础</p>
</li>
<li>
<p>对任务完成的好坏，给予一定的评判标准</p>
</li>
<li>
<p>通过分析经验数据，任务完成得更好了</p>
</li>
</ul>
</div>
</li>
</ul>
</div>
</div>
<div class="sect3">
<h4 id="_机器学习的开端">3.1.2. 机器学习的开端</h4>
<div class="ulist">
<ul>
<li>
<p>1952年，IBM的Arthur Samuel（被誉为“机器学习之父”）设计了一款可以学习的西洋跳棋程序。</p>
</li>
<li>
<p>它能通过观察棋子的走位来构建新的模型，并用其提高自己的下棋技巧。</p>
</li>
<li>
<p>Samuel和这个程序进行多场对弈后发现，随着时间的推移，程序的棋艺变得越来越好。</p>
</li>
</ul>
</div>
<div class="admonitionblock note">
<table>
<tr>
<td class="icon">
<i class="fa icon-note" title="Note"></i>
</td>
<td class="content">
Samuel九十岁去世，八十八岁还在写代码。。。
</td>
</tr>
</table>
</div>
</div>
<div class="sect3">
<h4 id="_机器学习的定义">3.1.3. 机器学习的定义</h4>
<div class="ulist">
<ul>
<li>
<p>机器学习(Machine Learning, ML)主要研究计算机系统对于特定任务的性能，逐步进行改善的算法和统计模型。</p>
</li>
<li>
<p>通过输入海量训练数据对模型进行训练，使模型掌握数据所蕴含的潜在规律，进而对新输入的数据进行准确的分类或预测。</p>
</li>
<li>
<p>是一门多领域交叉学科，涉及概率论、统计学、逼近论、凸优化、算法复杂度理论等多门学科。专门研究计算机怎样模拟或实现人类的学习行为，以获取新的知识或技能，重新组织已有的知识结构使之不断改善自身的性能。</p>
</li>
</ul>
</div>
</div>
<div class="sect3">
<h4 id="_机器学习的过程">3.1.4. 机器学习的过程</h4>
<div class="paragraph">
<p>海量数据 &#8594; 提炼规律 &#8594; 预测未来</p>
</div>
</div>
</div>
<div class="sect2">
<h3 id="_机器学习的分类">3.2. 机器学习的分类</h3>
<div class="ulist">
<ul>
<li>
<p>监督学习：提供数据并提供数据对应结果的机器学习过程。</p>
</li>
<li>
<p>无监督学习：提供数据并且不提供数据对应结果的机器学习过程。</p>
</li>
<li>
<p>强化学习：通过与环境交互并获取延迟返回进而改进行为的学习过程。</p>
</li>
</ul>
</div>
<div class="imageblock">
<div class="content">
<img src="images/class.png" alt="class">
</div>
</div>
<div class="sect3">
<h4 id="_监督学习">3.2.1. 监督学习</h4>
<div class="imageblock">
<div class="content">
<img src="images/supervised.png" alt="supervised">
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>监督学习（Supervised Learning）算法构建了包含输入和所需输出的一组数据的数学模型。这些数据称为训练数据，由一组训练样本组成。</p>
</li>
<li>
<p>监督学习主要包括分类和回归。</p>
</li>
<li>
<p>当输出被限制为有限的一组值（离散数值）时使用分类算法；当输出可以具有范围内的任何数值（连续数值）时使用回归算法。</p>
</li>
<li>
<p>相似度学习是和回归和分类都密切相关的一类监督机器学习，它的目标是使用相似性函数从样本中学习，这个函数可以度量两个对象之间的相似度或关联度。它在排名、推荐系统、视觉识别跟踪、人脸识别等方面有很好的应用场景。</p>
</li>
</ul>
</div>
<div class="paragraph">
<p><strong>例子</strong></p>
</div>
<table class="tableblock frame-all grid-all stretch">
<caption class="title">Table 1. 预测房价或者房屋出售情况</caption>
<colgroup>
<col style="width: 16.6666%;">
<col style="width: 16.6666%;">
<col style="width: 16.6666%;">
<col style="width: 16.6666%;">
<col style="width: 16.6666%;">
<col style="width: 16.667%;">
</colgroup>
<thead>
<tr>
<th class="tableblock halign-left valign-top">所在街区</th>
<th class="tableblock halign-left valign-top">房屋价格</th>
<th class="tableblock halign-left valign-top">住房面积</th>
<th class="tableblock halign-left valign-top">住房格局</th>
<th class="tableblock halign-left valign-top">是否学区</th>
<th class="tableblock halign-left valign-top">是否售出</th>
</tr>
</thead>
<tbody>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">海淀</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">7000000</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">120</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">三室一厅</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">是</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">是</p></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">朝阳</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">6000000</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">100</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">二室一厅</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">否</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">否</p></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">昌平</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">5000000</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">120</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">二室一厅</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">否</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">是</p></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">大兴</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">6500000</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">150</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">三室一厅</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">否</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">？</p></td>
</tr>
</tbody>
</table>
</div>
<div class="sect3">
<h4 id="_监督学习深入介绍">3.2.2. 监督学习深入介绍</h4>
<div class="ulist">
<ul>
<li>
<p>监督学习三要素</p>
</li>
<li>
<p>监督学习实现步骤</p>
</li>
<li>
<p>监督学习模型评估策略</p>
</li>
<li>
<p>分类和回归</p>
</li>
<li>
<p>监督学习模型求解算法</p>
</li>
</ul>
</div>
<div class="sect4">
<h5 id="_监督学习三要素">监督学习三要素</h5>
<div class="ulist">
<ul>
<li>
<p>模型（model）：总结数据的内在规律，用数学函数描述的系统</p>
</li>
<li>
<p>策略（strategy）：选取最优模型的评价准则</p>
</li>
<li>
<p>算法（algorithm）：选取最优模型的具体方法</p>
</li>
</ul>
</div>
</div>
<div class="sect4">
<h5 id="_监督学习实现步骤">监督学习实现步骤</h5>
<div class="ulist">
<ul>
<li>
<p>得到一个有限的训练数据集</p>
</li>
<li>
<p>确定包含所有学习模型的集合</p>
</li>
<li>
<p>确定模型选择的准则，也就是学习策略</p>
</li>
<li>
<p>实现求解最优模型的算法，也就是学习算法</p>
</li>
<li>
<p>通过学习算法选择最优模型</p>
</li>
<li>
<p>利用得到的最优模型，对新数据进行预测或分析</p>
</li>
</ul>
</div>
</div>
<div class="sect4">
<h5 id="_监督学习过程示例">监督学习过程示例</h5>
<div class="imageblock">
<div class="content">
<img src="images/luojihuigui7.png" alt="luojihuigui7">
</div>
</div>
<div class="imageblock">
<div class="content">
<img src="images/luojihuigui8.png" alt="luojihuigui8">
</div>
</div>
<div class="imageblock">
<div class="content">
<img src="images/model1.png" alt="model1">
</div>
</div>
</div>
</div>
<div class="sect3">
<h4 id="_模型评估策略">3.2.3. 模型评估策略</h4>
<div class="ulist">
<ul>
<li>
<p>模型评估</p>
<div class="ulist">
<ul>
<li>
<p>训练集和测试集</p>
</li>
<li>
<p>损失函数和经验风险</p>
</li>
<li>
<p>训练误差和测试误差</p>
</li>
</ul>
</div>
</li>
<li>
<p>模型选择</p>
<div class="ulist">
<ul>
<li>
<p>过拟合和欠拟合</p>
</li>
<li>
<p>正则化和交叉验证</p>
</li>
</ul>
</div>
</li>
</ul>
</div>
<div class="sect4">
<h5 id="_训练集和测试集">训练集和测试集</h5>
<div class="ulist">
<ul>
<li>
<p>我们将数据输入到模型中训练出了对应模型，但是模型的效果好不好呢？我们需要对模型的好坏进行评估</p>
</li>
<li>
<p>我们将用来训练模型的数据称为训练集，将用来测试模型好坏的集合称为测试集。</p>
</li>
<li>
<p>训练集：输入到模型中对模型进行训练的数据集合。</p>
</li>
<li>
<p>测试集：模型训练完成后测试训练效果的数据集合。</p>
</li>
</ul>
</div>
</div>
<div class="sect4">
<h5 id="_损失函数">损失函数</h5>
<div class="ulist">
<ul>
<li>
<p>损失函数用来衡量模型预测误差的大小。</p>
</li>
<li>
<p>定义：选取模型f为决策函数，对于给定的输入参数X，f(X)为预测结果，Y为真实结果；f(X)和Y之间可能会有偏差，我们就用一个损失函数（loss function）来度量预测偏差的程度，记作L(Y,f(X))</p>
</li>
<li>
<p>损失函数是系数的函数</p>
</li>
<li>
<p>损失函数值越小，模型就越好</p>
</li>
</ul>
</div>
</div>
<div class="sect4">
<h5 id="_常见损失函数">常见损失函数</h5>
<div class="ulist">
<ul>
<li>
<p>0-1损失函数</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[\begin{equation}
  L(Y,f(X)) =
    \begin{cases}
      1 &amp; Y \neq f(X) \\
      0 &amp; Y=f(X)
    \end{cases}
\end{equation}\]
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>平方损失函数</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[L(Y,f(X))=(Y-f(X))^2\]
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>绝对损失函数</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[L(Y,f(X))=\vert Y-f(X) \vert\]
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>对数损失函数</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[L(Y,P(Y \vert X))=-logP(Y \vert X)\]
</div>
</div>
</div>
<div class="sect4">
<h5 id="_经验风险">经验风险</h5>
<div class="ulist">
<ul>
<li>
<p>经验风险</p>
<div class="ulist">
<ul>
<li>
<p>模型f(X)关于训练数据集的平均损失称为经验风险（empirial risk），记作\(R_{emp}\)</p>
</li>
</ul>
</div>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[R_{emp}(f)=\frac{1}{N}\sum_{i=1}^N L(y_i, f(x_i))\]
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>经验风险最小化（Empirical Risk Minimization，ERM）</p>
<div class="ulist">
<ul>
<li>
<p>这一策略认为，经验风险最小的模型就是最优的模型</p>
</li>
<li>
<p>样本足够大时，ERM有很好的学习效果，因为有足够多的“经验”</p>
</li>
<li>
<p>样本较小时，ERM就会出现一些问题</p>
</li>
</ul>
</div>
</li>
</ul>
</div>
</div>
<div class="sect4">
<h5 id="_训练误差和测试误差">训练误差和测试误差</h5>
<div class="ulist">
<ul>
<li>
<p>训练误差</p>
<div class="ulist">
<ul>
<li>
<p>训练误差（training error）是关于训练集的平均损失。</p>
</li>
</ul>
</div>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[R_{emp}(\hat{f})=\frac{1}{N}\sum_{i=1}^N L(y_i, \hat{f}(x_i))\]
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>训练误差的大小，可以用来判断给定问题是否容易学习，但本质上并不重要</p>
<div class="ulist">
<ul>
<li>
<p>测试误差</p>
</li>
</ul>
</div>
</li>
<li>
<p>测试误差（testing error）是关于测试集的平均损失。</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[e_{test}(f)=\frac{1}{N'}\sum_{i=1}^{N'} L(y_i, \hat{f}(x_i))\]
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>测试误差真正反映了模型对未知数据的预测能力，这种能力一般被称为泛化能力。</p>
</li>
</ul>
</div>
</div>
<div class="sect4">
<h5 id="_过拟合和欠拟合">过拟合和欠拟合</h5>
<div class="imageblock">
<div class="content">
<img src="images/overfit.png" alt="overfit">
</div>
</div>
<div class="paragraph">
<p><strong>欠拟合</strong></p>
</div>
<div class="ulist">
<ul>
<li>
<p>模型没有很好地捕捉到数据特征，特征集过小，导致模型不能很好地拟合数据，称之为欠拟合（under-fitting）</p>
</li>
<li>
<p>欠拟合的本质是对数据的特征“学习”得不够</p>
</li>
<li>
<p>例如，想分辨一只猫，只给出了四条腿、两只眼、有尾巴这三个特征，那么由此训练出来的模型根本无法分辨猫</p>
</li>
</ul>
</div>
<div class="paragraph">
<p><strong>过拟合</strong></p>
</div>
<div class="ulist">
<ul>
<li>
<p>把训练数据学习的太彻底，以至于把噪声数据的特征也学习到了，特征集过大，这样就会导致在后期测试的时候不能够很好地识别数据，即不能正确的分类，模型泛化能力太差，称之为过拟合（over-fitting）。</p>
</li>
<li>
<p>例如，想分辨一只猫，给出了四条腿、两只眼、一条尾巴、叫声、颜色，能够捕捉老鼠、喜欢吃鱼、&#8230;&#8203;，然后恰好所有的训练数据的猫都是白色，那么这个白色是一个噪声数据，会干扰判断，结果模型把颜色是白色也学习到了，而白色是局部样本的特征，不是全局特征，就造成了输入一个黑猫的数据，判断出不是猫。</p>
</li>
</ul>
</div>
<div class="paragraph">
<p><strong>例子</strong></p>
</div>
<div class="imageblock">
<div class="content">
<img src="images/fitting.png" alt="fitting">
</div>
</div>
<div class="stemblock">
<div class="content">
\[f_M(x,w)=w_0+w_1x+w_2x^2+ \dots + w_Mx^M=\sum_{j=0}^Mw_jx^j\]
</div>
</div>
<div class="paragraph">
<p>以上可以看到，当\(M=0,M=1\)时，是明显的欠拟合状态，有些数据没有拟合到。而当\(M=3\)时，虽然没有和样本点完全吻合，却是拟合的最好的一条曲线。而当\(M=9\)时，曲线严格吻合样本点，但却是严重的过拟合，为什么？因为在拟合或者说机器学习的过程中，我们把噪声也拟合进去了，所以造成了过拟合。</p>
</div>
</div>
</div>
<div class="sect3">
<h4 id="_模型的选择">3.2.4. 模型的选择</h4>
<div class="ulist">
<ul>
<li>
<p>当模型复杂度增大时，训练误差会逐渐减小并趋向于0；而测试误差会先减小，达到最小值之后再增大</p>
</li>
<li>
<p>当模型复杂度过大时，就会发生过拟合；所以模型复杂度应适当</p>
</li>
</ul>
</div>
<div class="imageblock">
<div class="content">
<img src="images/model.png" alt="model">
</div>
</div>
</div>
<div class="sect3">
<h4 id="_正则化">3.2.5. 正则化</h4>
<div class="ulist">
<ul>
<li>
<p>结构风险最小化（Structural Risk Minimization，SRM）</p>
<div class="ulist">
<ul>
<li>
<p>是在ERM基础上，为了防止过拟合而提出来的策略</p>
</li>
<li>
<p>在经验风险上加上表示模型复杂度的正则化项（regularizer），或者叫惩罚项</p>
</li>
<li>
<p>正则化项一般是模型复杂度的单调递增函数，即模型越复杂，正则化值越大</p>
</li>
</ul>
</div>
</li>
<li>
<p>结构风险最小化的典型实现是正则化（regularization）</p>
<div class="ulist">
<ul>
<li>
<p>形式：</p>
</li>
</ul>
</div>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[\min_{f \in F} \frac{1}{N}\sum_{i=1}^N L(y_i,f(x_i))+\lambda J(f)\]
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>第一项是经验风险，第二项\(J(f)\)是正则化项，\(\lambda \geq 0\)是调整两者关系的系数</p>
</li>
<li>
<p>正则化项可以取不同的形式，比如，特征向量的\(L_1\)范数或\(L_2\)范数</p>
</li>
</ul>
</div>
<div class="paragraph">
<p>例如：</p>
</div>
<div class="stemblock">
<div class="content">
\[f_M(x,w)=w_0+w_1x+w_2x^2+ \dots + w_Mx^M=\sum_{j=0}^Mw_jx^j\]
</div>
</div>
<div class="paragraph">
<p>的正则化项可以设置为</p>
</div>
<div class="stemblock">
<div class="content">
\[\sqrt{\sum_{j=0}^Mw_j^2}\]
</div>
</div>
<div class="paragraph">
<p>这样就可以很好的惩罚过拟合，因为一旦过拟合，系数将会非常的大，而这就造成了结构风险也会很大。</p>
</div>
<div class="sect4">
<h5 id="_奥卡姆剃刀">奥卡姆剃刀</h5>
<div class="ulist">
<ul>
<li>
<p>奥卡姆剃刀(Occam‘s razor)原理：如无必要，勿增实体</p>
</li>
<li>
<p>正则化符合奥卡姆剃刀原理。它的思想是：在所有可能选择的模型中，我们应该选择能够很好地解释已知数据并且十分简单的模型</p>
</li>
<li>
<p>如果简单的模型已经够用，我们不应该一味地追求更小的训练误差，而把模型变得越来越复杂</p>
</li>
</ul>
</div>
</div>
</div>
<div class="sect3">
<h4 id="_交叉验证">3.2.6. 交叉验证</h4>
<div class="ulist">
<ul>
<li>
<p>数据集划分</p>
<div class="ulist">
<ul>
<li>
<p>如果样本数据充足，一种简单方法是随机将数据集切成三部分：训练集（training set）、验证集（validation set）和测试集（test set）</p>
</li>
<li>
<p>训练集用于训练模型，验证集用于模型选择，测试集用于学习方法评估</p>
</li>
</ul>
</div>
</li>
<li>
<p>数据不充足时，可以重复地利用数据——交叉验证（cross validation）</p>
<div class="ulist">
<ul>
<li>
<p>简单交叉验证</p>
<div class="ulist">
<ul>
<li>
<p>数据随机分为两部分，如70%作为训练集，剩下30%作为测试集</p>
</li>
<li>
<p>训练集在不同的条件下（比如参数个数）训练模型，得到不同的模型</p>
</li>
<li>
<p>在测试集上评价各个模型的测试误差，选出最优模型</p>
</li>
</ul>
</div>
</li>
<li>
<p>S折交叉验证</p>
<div class="ulist">
<ul>
<li>
<p>将数据随机切分为S个互不相交、相同大小的子集；S-1个做训练集，剩下一个做测试集</p>
</li>
<li>
<p>重复进行训练集、测试集的选取，有S种可能的选择</p>
</li>
</ul>
</div>
</li>
<li>
<p>留一交叉验证</p>
</li>
</ul>
</div>
</li>
</ul>
</div>
</div>
<div class="sect3">
<h4 id="_分类和回归">3.2.7. 分类和回归</h4>
<div class="ulist">
<ul>
<li>
<p>监督学习问题主要可以划分为两类，即分类问题和回归问题</p>
<div class="ulist">
<ul>
<li>
<p>分类问题预测数据属于哪一类别。 —— 离散</p>
</li>
<li>
<p>回归问题根据数据预测一个数值。 —— 连续</p>
</li>
</ul>
</div>
</li>
<li>
<p>通俗地讲，分类问题就是预测数据属于哪一种类型，就像上面的房屋出售预测，通过大量数据训练模型，然后去预测某个给定房屋能不能出售出去，属于能够出售类型还是不能出售类型。</p>
</li>
<li>
<p>回归问题就是预测一个数值，比如给出房屋一些特征，预测房价</p>
</li>
<li>
<p>如果将上面的房屋出售的问题改为预测房屋出售的概率，得到的结果将不再是可以售出（1）和不能售出（0），将会是一个连续的数值，例如 0.5，这就变成了一个回归问题</p>
</li>
</ul>
</div>
<div class="sect4">
<h5 id="_分类问题">分类问题</h5>
<div class="imageblock">
<div class="content">
<img src="images/fenlei.png" alt="fenlei">
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>在监督学习中，当输出变量 Y 取有限个离散值时，预测问题就成了分类（classification）问题</p>
</li>
<li>
<p>监督学习从数据中学习一个分类模型或分类决策函数，称为分类器（classifier）；分类器对新的输入进行预测，称为分类</p>
</li>
<li>
<p>分类问题包括学习和分类两个过程。学习过程中，根据已知的训练数据集利用学习方法学习一个分类器；分类过程中，利用已习得的分类器对新的输入实力进行分类</p>
</li>
<li>
<p>分类问题可以用很多学习方法来解决，比如k近邻、决策树、感知机、逻辑斯谛回归、支撑向量机、朴素贝叶斯法、神经网络等</p>
</li>
</ul>
</div>
</div>
<div class="sect4">
<h5 id="_精确率和召回率">精确率和召回率</h5>
<div class="ulist">
<ul>
<li>
<p>评价分类器性能的指标一般是分类准确率（accuracy），它定义为分类器对测试集正确分类的样本数与总样本数之比</p>
</li>
<li>
<p>对于二类分类问题，常用的评价指标是精确率（precision）与召回率（recall）</p>
</li>
<li>
<p>通常以关注的类为正类，其它为负类，按照分类器在测试集上预测的正确与否，会有四种情况出现，它们的总数分别记作：</p>
<div class="ulist">
<ul>
<li>
<p>TP：将正类预测为正类的数目</p>
</li>
<li>
<p>FN：将正类预测为负类的数目</p>
</li>
<li>
<p>FP：将负类预测为正类的数目</p>
</li>
<li>
<p>TN：将负类预测为负类的数目</p>
</li>
</ul>
</div>
</li>
<li>
<p>精确率</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[P=\frac{TP}{TP+FP}\]
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>精确率指的是“所有预测为正类的数据中，预测正确的比例”</p>
<div class="ulist">
<ul>
<li>
<p>召回率</p>
</li>
</ul>
</div>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[R=\frac{TP}{TP+FN}\]
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>召回率指的是“所有实际为正类的数据中，被正确预测找出的比例”</p>
</li>
</ul>
</div>
</div>
<div class="sect4">
<h5 id="_回归问题">回归问题</h5>
<div class="imageblock">
<div class="content">
<img src="images/huigui.png" alt="huigui">
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>回归问题用于预测输入变量和输出变量之间的关系</p>
</li>
<li>
<p>回归模型就是表示从输入变量到输出变量之间映射的函数</p>
</li>
<li>
<p>回归问题的学习等价于函数拟合：选择一条函数曲线，使其很好地拟合已知数据，并且能够很好地预测未知数据</p>
</li>
<li>
<p>回归问题的分类</p>
<div class="ulist">
<ul>
<li>
<p>按照输入变量的个数：一元回归和多元回归</p>
</li>
<li>
<p>按照模型类型：线性回归和非线性回归</p>
</li>
<li>
<p>回归学习的损失函数 —— 平方损失函数</p>
</li>
<li>
<p>如果选取平方损失函数作为损失函数，回归问题可以用著名的最小二乘法（least squares）来求解</p>
</li>
</ul>
</div>
</li>
</ul>
</div>
</div>
</div>
<div class="sect3">
<h4 id="_模型求解算法学习算法">3.2.8. 模型求解算法（学习算法）</h4>
<div class="ulist">
<ul>
<li>
<p>梯度下降算法</p>
</li>
<li>
<p>牛顿法和拟牛顿法</p>
</li>
</ul>
</div>
<div class="sect4">
<h5 id="_梯度下降算法">梯度下降算法</h5>
<div class="ulist">
<ul>
<li>
<p>梯度下降（gradient descent）是一种常用的一阶优化方法，是求解无约束优化问题最简单、最经典的方法之一</p>
</li>
<li>
<p>梯度方向：函数变化增长最快的方向（变量沿此方向变化时函数增长最快）</p>
</li>
<li>
<p>负梯度方向：函数变化减少最快的方向（变量沿此方向变化时函数减少最快）</p>
</li>
<li>
<p>损失函数是系数的函数，那么如果系数沿着损失函数的负梯度方向变化，此时损失函数减少最快，能够以最快速度下降到极小值</p>
</li>
<li>
<p>沿着负梯度方向迭代，迭代后的\(\theta\)使损失函数\(J(\theta)\)更小：</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[\theta = \theta - \alpha  \frac{\partial J(\theta)}{\partial \theta}\]
</div>
</div>
<div class="imageblock">
<div class="content">
<img src="images/tiduxiajiang.png" alt="tiduxiajiang">
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>比如我们在一座大山上的某处位置，由于我们不知道怎么下山，于是决定走一步算一步，也就是在每走到一个位置的时候，求解当前位置的梯度，沿着梯度的负方向，也就是当前最陡峭的位置向下走一步，然后继续求解当前位置梯度，向这一步所在位置沿着最陡峭最易下山的位置走一步。这样一步步的走下去，一直走到觉得我们已经到了山脚。当然这样走下去，有可能我们不能走到山脚，而是到了某一个局部的山谷处。</p>
</li>
<li>
<p>从上面的解释可以看出，梯度下降不一定能够找到全局的最优解，有可能是一个局部最优解</p>
</li>
<li>
<p>如果损失函数是凸函数，梯度下降法得到的解就一定是全局最优解</p>
</li>
</ul>
</div>
</div>
</div>
<div class="sect3">
<h4 id="_牛顿法和拟牛顿法">3.2.9. 牛顿法和拟牛顿法</h4>
<div class="sect4">
<h5 id="_牛顿法">牛顿法</h5>
<div class="paragraph">
<p>迭代公式：</p>
</div>
<div class="stemblock">
<div class="content">
\[x^{(k+1)}=x^{(k)}-H^{-1}_k g_k\]
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>其中\(g_k=g(x^{(k)})=\nabla f(x^{(k)})\)是f(x) 的梯度向量在\(x^{(k)}\)的值，</p>
</li>
<li>
<p>\(H(x^{(k)})\)是f(x)的海塞矩阵在\(x^{(k)}\)的值</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[H(x)=[\frac{\partial^2 f}{\partial x_i \partial x_j}]_{n \times n}\]
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>梯度下降法只考虑了一阶导数，而牛顿法考虑了二阶导数，因此收敛速度更快</p>
</li>
</ul>
</div>
</div>
<div class="sect4">
<h5 id="_拟牛顿法">拟牛顿法</h5>
<div class="ulist">
<ul>
<li>
<p>牛顿法需要求解目标函数的海赛矩阵的逆矩阵，计算比较复杂</p>
</li>
<li>
<p>拟牛顿法通过正定矩阵近似海赛矩阵的逆矩阵，从而大大简化了计算过程</p>
</li>
</ul>
</div>
</div>
</div>
</div>
</div>
</div>
<div class="sect1">
<h2 id="_机器学习模型简介">4. 机器学习模型简介</h2>
<div class="sectionbody">
<div class="paragraph">
<p>主要内容</p>
</div>
<div class="paragraph">
<p><strong>监督学习</strong></p>
</div>
<div class="ulist">
<ul>
<li>
<p>回归模型</p>
<div class="ulist">
<ul>
<li>
<p>线性回归</p>
</li>
</ul>
</div>
</li>
<li>
<p>分类模型</p>
<div class="ulist">
<ul>
<li>
<p>感知机</p>
</li>
<li>
<p>K近邻（KNN）</p>
</li>
<li>
<p>决策树</p>
</li>
<li>
<p>逻辑斯蒂回归</p>
</li>
</ul>
</div>
</li>
</ul>
</div>
<div class="paragraph">
<p><strong>无监督学习</strong></p>
</div>
<div class="ulist">
<ul>
<li>
<p>聚类</p>
<div class="ulist">
<ul>
<li>
<p>K均值聚类</p>
</li>
</ul>
</div>
</li>
</ul>
</div>
<div class="sect2">
<h3 id="_使用anaconda">4.1. 使用Anaconda</h3>
<div class="paragraph">
<p><a href="https://zhuanlan.zhihu.com/p/32925500">点击链接学习如何安装和使用Anaconda</a></p>
</div>
</div>
<div class="sect2">
<h3 id="_线性回归">4.2. 线性回归</h3>
<div class="ulist">
<ul>
<li>
<p>线性回归（linear regression）是一种线性模型，它假设输入变量x和单个输出变量y之间存在线性关系</p>
</li>
<li>
<p>具体来说，利用线性回归模型，可以从一组输入变量x的线性组合中，计算输出变量y</p>
</li>
<li>
<p>一元线性回归(只有一个特征\(x\))</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[y=wx+b\]
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>多元线性回归(有多个特征\(x_0,x_1,\dots,x_n\))</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[f(x) = w_1 x_1 + w_2 x_2 + \dots + w_n x_n + b\]
</div>
</div>
<div class="imageblock">
<div class="content">
<img src="images/xianxinghuigui.png" alt="xianxinghuigui">
</div>
</div>
<div class="admonitionblock note">
<table>
<tr>
<td class="icon">
<i class="fa icon-note" title="Note"></i>
</td>
<td class="content">
一元线性回归就是从上图中的一堆散点图，拟合出一条直线\(y=wx+b\)。这里的拟合其实就是机器学习。散点为训练数据，我们要从训练数据中学习出一条直线。
</td>
</tr>
</table>
</div>
<div class="paragraph">
<p>这里的模型就是线性回归模型。那么机器学习的算法呢？</p>
</div>
<div class="ulist">
<ul>
<li>
<p>最小二乘法</p>
</li>
<li>
<p>梯度下降法</p>
</li>
</ul>
</div>
<div class="admonitionblock note">
<table>
<tr>
<td class="icon">
<i class="fa icon-note" title="Note"></i>
</td>
<td class="content">
注意我们这里还没有讲监督学习三要素中的策略呢！
</td>
</tr>
</table>
</div>
<div class="sect3">
<h4 id="_损失函数_2">4.2.1. 损失函数</h4>
<div class="paragraph">
<p>这里使用平方损失函数，并且没有正则化项。</p>
</div>
<div class="stemblock">
<div class="content">
\[L(w,b) = \sum_{i=1}^m (wx_i + b - y_i)^2\]
</div>
</div>
</div>
<div class="sect3">
<h4 id="_学习算法">4.2.2. 学习算法</h4>
<div class="paragraph">
<p>求解\(w,b\)使得平方损失函数取得最小值。</p>
</div>
<div class="ulist">
<ul>
<li>
<p>最小二乘法</p>
</li>
<li>
<p>梯度下降法</p>
</li>
</ul>
</div>
</div>
<div class="sect3">
<h4 id="_最小二乘法">4.2.3. 最小二乘法</h4>
<div class="ulist">
<ul>
<li>
<p>基于均方误差最小化来进行模型求解的方法称为“最小二乘法”（least square method）。</p>
</li>
<li>
<p>它的主要思想就是选择未知参数，使得理论值与观测值之差的平方和达到最小。</p>
</li>
</ul>
</div>
<div class="imageblock">
<div class="content">
<img src="images/zuixiaoercheng.png" alt="zuixiaoercheng">
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>我们假设输入属性（特征）的数目只有一个：就是\(f(x)=wx+b\)中的\(x\)。我们的训练数据是\((x_1,y_1),(x_2,y_2),\dots,(x_n,y_n)\)。要从这些训练数据中学习一个模型出来，使得：\(f(x_i)=wx_i+b, f(x_i) \approx y_i\)。</p>
</li>
<li>
<p>在线性回归中，最小二乘法就是试图找到一条直线，使所有样本到直线上的欧式距离之和最小。</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[\begin{align}
(w^*,b^*) &amp;= \arg \min_{(w,b)}\sum_{i=1}^m (f(x_i)-y_i)^2 \\
          &amp;= \arg \min_{(w,b)}\sum_{i=1}^m (wx_i+b-y_i)^2
\end{align}\]
</div>
</div>
<div class="admonitionblock note">
<table>
<tr>
<td class="icon">
<i class="fa icon-note" title="Note"></i>
</td>
<td class="content">
以上就是监督学习三要素中的策略！
</td>
</tr>
</table>
</div>
<div class="sect4">
<h5 id="_最小二乘法求解过程">最小二乘法求解过程</h5>
<div class="ulist">
<ul>
<li>
<p>求解\(w\)和\(b\)，使得\(E_{(w,b)}=\sum_{i=1}^m (wx_i+b-y_i)^2\)最小化的过程，称为线性回归模型的“最小二乘估计”。</p>
</li>
<li>
<p>将\(E_{(w,b)}\)分别对\(w\)和\(b\)进行求导，可以得到：</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[\begin{align}
\frac{\partial E_{(w,b)}}{\partial w}
&amp;= \frac{\partial ((wx_1+b-y_1)^2+ \dots + (wx_m+b-y_m)^2)}{\partial w} \\
&amp;= 2(wx_1+b-y_1)x_1+ \dots +2(wx_m+b-y_m)x_m \\
&amp;= 2(wx_1^2-(y_1-b)x_1) + \dots + 2(wx_m^2-(y_m-b)x_m) \\
&amp;= 2(w \sum_{i=1}^m x_i^2 - \sum_{i=1}^m (y_i-b)x_i) \\

\frac{\partial E_{(w,b)}}{b}
&amp;= \frac{\partial ((wx_1+b-y_1)^2+ \dots + (wx_m+b-y_m)^2)}{\partial w} \\
&amp;= 2(wx_1+b-y_1) + \dots + 2(wx_m+b-y_m) \\
&amp;= 2(mb-\sum_{i=1}^m (y_i-wx_i))
\end{align}\]
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>令偏导数都为0，可以得到：</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[2(mb-\sum_{i=1}^m (y_i-wx_i)) = 0 \\
b = \frac{1}{m}\sum_{i=1}^m (y_i-wx_i)\]
</div>
</div>
<div class="paragraph">
<p>将\(b\)代入到第一个式子中，可以解得\(w\)的值，然后再将\(w\)的值代入到b的表达式，即可解得\(b\)。</p>
</div>
<div class="stemblock">
<div class="content">
\[w = \frac{\sum_{i=1}^m y_i(x_i-\bar{x})}{\sum_{i=1}^m x_i^2-\frac{1}{m}(\sum_{i=1}^m x_i)^2} \\
b = \frac{1}{m}\sum_{i=1}^m (y_i-wx_i)\]
</div>
</div>
<div class="paragraph">
<p>其中：\(\bar{x}=\frac{1}{m}\sum_{i=1}^m x_i\)</p>
</div>
</div>
</div>
<div class="sect3">
<h4 id="_梯度下降法求解过程">4.2.4. 梯度下降法求解过程</h4>
<div class="paragraph">
<p>求解\(w\)和\(b\)，使得\(E_{(w,b)}=\sum_{i=1}^m (wx_i+b-y_i)^2\)达到极小值，也可以使用梯度下降法。</p>
</div>
<div class="paragraph">
<p>梯度为：</p>
</div>
<div class="stemblock">
<div class="content">
\[\nabla E =
[\frac{\partial}{\partial w} E_{(w,b)}, \frac{\partial}{\partial b} E_{(w,b)}] \\
\frac{\partial E_{(w,b)}}{\partial w} = 2(w \sum_{i=1}^m x_i^2 - \sum_{i=1}^m (y_i-b)x_i) \\
\frac{\partial E_{(w,b)}}{b} = 2(mb-\sum_{i=1}^m (y_i-wx_i))\]
</div>
</div>
<div class="admonitionblock note">
<table>
<tr>
<td class="icon">
<i class="fa icon-note" title="Note"></i>
</td>
<td class="content">
这里不是令偏导数为零了，而是准备梯度下降。
</td>
</tr>
</table>
</div>
<div class="stemblock">
<div class="content">
\[w_{next} := w_{current} - \alpha \frac{\partial E}{\partial w} \\
b_{next} := b_{current} - \alpha \frac{\partial E}{\partial b}\]
</div>
</div>
<div class="admonitionblock note">
<table>
<tr>
<td class="icon">
<i class="fa icon-note" title="Note"></i>
</td>
<td class="content">
以上，\(w,b\)的初始值，\(\alpha\)的值，以及迭代的次数，都是超参数。因为不是机器学习学习出来的参数。
</td>
</tr>
</table>
</div>
<div class="ulist">
<ul>
<li>
<p>\(\alpha\)在梯度下降算法中被称作为学习率或者步长</p>
</li>
<li>
<p>这意味着我们可以通过\(\alpha\)来控制每一步走的距离，以保证不要走太快，错过了最低点；同时也要保证收敛速度不要太慢</p>
</li>
<li>
<p>所以\(\alpha\)的选择在梯度下降法中往往是很重要的，不能太大也不能太小</p>
</li>
</ul>
</div>
<div class="imageblock">
<div class="content">
<img src="images/xuexilv.png" alt="xuexilv">
</div>
</div>
</div>
<div class="sect3">
<h4 id="_梯度下降法和最小二乘法">4.2.5. 梯度下降法和最小二乘法</h4>
<div class="ulist">
<ul>
<li>
<p>相同点</p>
<div class="ulist">
<ul>
<li>
<p>本质和目标相同：两种方法都是经典的学习算法，在给定已知数据的前提下利用求导算出一个模型（函数），使得损失函数最小，然后对给定的新数据进行估算预测</p>
</li>
</ul>
</div>
</li>
<li>
<p>不同点</p>
<div class="ulist">
<ul>
<li>
<p>损失函数：梯度下降可以选取其它损失函数，而最小二乘一定是平方损失函数</p>
</li>
<li>
<p>实现方法：最小二乘法是直接求导找出全局最小；而梯度下降是一种迭代法</p>
</li>
<li>
<p>效果：最小二乘找到的一定是全局最小，但计算繁琐，且复杂情况下未必有解；梯度下降迭代计算简单，但找到的一般是局部最小，只有在目标函数是凸函数时才是全局最小；到最小点附近时收敛速度会变慢，且对初始点的选择极为敏感</p>
</li>
</ul>
</div>
</li>
</ul>
</div>
</div>
</div>
<div class="sect2">
<h3 id="_感知机perceptron">4.3. 感知机（Perceptron）</h3>
<div class="sect3">
<h4 id="_感知机的定义">4.3.1. 感知机的定义</h4>
<div class="paragraph">
<p>感知机是机器学习应用中分类的最简单的一种算法。如下图所示：感知机划分超平面</p>
</div>
<div class="imageblock">
<div class="content">
<img src="images/ganzhiji1.png" alt="ganzhiji1">
</div>
</div>
<div class="paragraph">
<p>感知机是二分类的线性模型，输入是实例的特征向量，输出是实例的类别；假设训练的数据集是线性可分的，感知机的目标就是求一个能够将训练集的正负样本完全正确分离开的超平面(也就是上图中所示的那些将蓝、黄数据点完全分离开的直线)。但是如果这些数据是非线性可分的，这个超平面是无法获取的。上图的坐标轴，横坐标为\(X_1\)，纵坐标为\(X_2\)。图中的每一个点都由\((X_1,X_2)\)所决定。举个实例：有一批零件，判断零件是否合格有两个重要点，长度和重量。\(X_1\)表示长度，\(X_2\)表示重量，上图的两条黑线表示零件的长度均值和重量均值。只有当长度和重量都满足一定条件，该零件才为合格品。都不满足一定条件，视为不可修复的劣质品，直接丢弃。那么机器学习如何学习到这个规则呢？我们在代码实现的时候，拿到手的是所有样本的信息\((X_1,X_2)\)和标签(0或1)，标签里面0表示不合格品，1表示合格品。简单的说就是图片上黄色和蓝色的点。根据我们手上的这些点，我们需要找到一条直线将上面的点完美的分开。这样的直线我们可以找到很多条，那么哪一条才是最好的呢？实际上，感知机只是一个二分类的问题，无法找到一条最佳的直线，只需要能把所有的点都分开就好。我们设定损失函数为所有分错的点和直线的距离求和，然后训练，使这段求和的数值最小(最优的情况是0，因为0代表完全分开了)，那么这条直线就满足我们的条件，就是我们所找的。</p>
</div>
</div>
<div class="sect3">
<h4 id="_感知机的数学原理">4.3.2. 感知机的数学原理</h4>
<div class="paragraph">
<p>首先，点\(P(x_0,y_0)\)到直线\(Ax+By+C=0\)的距离为：</p>
</div>
<div class="stemblock">
<div class="content">
\[d=\frac{Ax_0+By_0+C}{\sqrt{A^2+B^2}}\]
</div>
</div>
<div class="paragraph">
<p>类似的：设超平面公式为：\(h=wx+b\)，其中\(w=(w_0,w_1,w_2,\dots,w_n),x=(x_0,x_1,x_2,\dots,x_n)\)。其中样本点\(x'\)到超平面的距离为：</p>
</div>
<div class="stemblock">
<div class="content">
\[d=\frac{wx'+b}{\parallel w \parallel}\]
</div>
</div>
<div class="paragraph">
<p>那么这个超平面为什么设置为\(wx+b\)呢？它和我们常见的\(ax+b\)有什么区别呢？</p>
</div>
<div class="paragraph">
<p>本质没啥区别，\(ax+b\)是二维中的，\(wx+b\)是高维中的。就看你的理解啦，简单的来说，\(wx+b\)是一个\(n\)维空间中的超平面\(S\)，其中\(w\)是超平面的法向量，\(b\)是超平面的截距，这个超平面将特征空间划分成两部分，位于两部分的点分别被分为正负两类，所以，超平面S称为分离超平面。其中\(w=(w_0,w_1,w_2,\dots,w_n),x=(x_0,x_1,x_2,\dots,x_n)\)。</p>
</div>
<div class="paragraph">
<p>细节：</p>
</div>
<div class="paragraph">
<p>\(w\)是超平面的法向量：对于一个平面来说\(w\)就是这么定义的。数学上就这么定义的。</p>
</div>
<div class="paragraph">
<p>\(b\)是超平面的截距：可以按照二维中的\(ax+b\)理解。</p>
</div>
<div class="paragraph">
<p>特征空间：也就是整个\(n\)维空间，样本的每个属性都叫一个特征，特征空间的意思就是在这个空间中可以找到样本所有的属性组合。</p>
</div>
</div>
<div class="sect3">
<h4 id="_感知机的模型">4.3.3. 感知机的模型</h4>
<div class="imageblock">
<div class="content">
<img src="images/ganzhiji2.png" alt="ganzhiji2">
</div>
</div>
<div class="paragraph">
<p>感知机的模型：输入空间—&gt;输出空间：</p>
</div>
<div class="stemblock">
<div class="content">
\[f(x)=sign(wx+b), 其中, sign(x)=
\begin{cases}
  -1 &amp; x &lt; 0 \\
  1 &amp; x \ge 0
\end{cases}\]
</div>
</div>
<div class="paragraph">
<p>sign函数很简单，当x大于等于0，sign输出1，否则输出-1。那么往前想一下，\(wx+b\)如果大于等于0，\(f(x)\)就等于1，反之\(f(x)\)等于-1。</p>
</div>
</div>
<div class="sect3">
<h4 id="_感知机的损失函数">4.3.4. 感知机的损失函数</h4>
<div class="paragraph">
<p>我们定义样本\((x_i,y_i)\)，如果上面的距离\(d &gt; 0\)，则\(y_i=1\)；如果\(d &lt; 0\),则\(y_i=-1\)，这样取\(y\)有一个好处，就是方便定义损失函数。优化的目标：期望使误分类的所有样本，到超平面的距离之和最小。</p>
</div>
<div class="paragraph">
<p>所以定义损失函数为：</p>
</div>
<div class="stemblock">
<div class="content">
\[L(w,b)=-\frac{1}{\parallel w \parallel} \sum_{x_i \in M}y_i(wx_i+b)\]
</div>
</div>
<div class="paragraph">
<p>其中M集合就是误分类点的集合。</p>
</div>
<div class="paragraph">
<p>不考虑前面的系数，感知机模型的损失函数为：</p>
</div>
<div class="stemblock">
<div class="content">
\[L(w,b)=-\sum_{x_i \in M}y_i(wx_i+b)\]
</div>
</div>
</div>
<div class="sect3">
<h4 id="_感知机学习算法">4.3.5. 感知机学习算法</h4>
<div class="paragraph">
<p>感知机学习算法是对于上述损失函数进行极小化，求得\(w\)和\(b\)。这里使用随机梯度下降法(SGD)，因为误分类的M集合里面的样本才能参加损失函数的优化。</p>
</div>
<div class="paragraph">
<p>目标函数如下：</p>
</div>
<div class="stemblock">
<div class="content">
\[L(w,b)=\arg \min_{w,b}(-\sum_{x_i \in M}y_i(wx_i+b))\]
</div>
</div>
<div class="paragraph">
<p><strong>算法步骤</strong></p>
</div>
<div class="exampleblock">
<div class="content">
<div class="paragraph">
<p>输入：训练数据集\(T=(x_1,y_1),(x_2,y_2),\dots,(x_N,y_N),y_i \in \{-1,+1\},学习率 \eta (0 &lt; \eta &lt; 1)\)</p>
</div>
<div class="paragraph">
<p>输出：\(w,b\)；感知机模型\(f(x)=sign(wx+b)\)</p>
</div>
<div class="olist arabic">
<ol class="arabic">
<li>
<p>赋初值\(w_0,b_0\)</p>
</li>
<li>
<p>选取数据点\((x_i,y_i)\)</p>
</li>
<li>
<p>判断该数据点是否为当前模型的误分类点，即判断若\(y_i(wx_i+b) \le 0\)则更新：</p>
</li>
</ol>
</div>
<div class="stemblock">
<div class="content">
\[w \leftarrow w + \eta y_i x_i \\
b \leftarrow b + \eta y_i\]
</div>
</div>
<div class="olist arabic">
<ol class="arabic" start="4">
<li>
<p>转到2，直到训练集中没有误分类点。</p>
</li>
</ol>
</div>
</div>
</div>
</div>
</div>
<div class="sect2">
<h3 id="_逻辑回归logistic_regression">4.4. 逻辑回归（Logistic Regression）</h3>
<div class="sect3">
<h4 id="_直观理解">4.4.1. 直观理解</h4>
<div class="admonitionblock note">
<table>
<tr>
<td class="icon">
<i class="fa icon-note" title="Note"></i>
</td>
<td class="content">
逻辑回归虽然叫回归，但其实是分类算法。
</td>
</tr>
</table>
</div>
<div class="paragraph">
<p>我们还记不记得感知机一节中给出的这张图？还记得感知机是怎么工作的吗？</p>
</div>
<div class="imageblock">
<div class="content">
<img src="images/luojihuigui.png" alt="luojihuigui">
</div>
</div>
<div class="paragraph">
<p>感知机生成一个超平面（在二维中表现为一条直线），这个超平面可以将图中的两类点区分开。像下面这张图，图中的所有直线都可以作为感知机的超平面（因为它们都能把训练集中的点完美地分开）。</p>
</div>
<div class="imageblock">
<div class="content">
<img src="images/luojihuigui2.png" alt="luojihuigui2">
</div>
</div>
<div class="paragraph">
<p>但是感知机存在一个很重要的问题，我们只用\(sign(wx+b)\)输出的\(+1\)和\(-1\)来判断点的类别是不是太简单了？是不是有点生硬的感觉。</p>
</div>
<div class="paragraph">
<p>这么简单的判别方式真的会很有效吗？</p>
</div>
<div class="paragraph">
<p>当然了，虽然我们已经程序测试过正确率很高，但总是让人有点担心是否在很多情况下都能很好地工作。事实上我们从小到大一直会听到一些升学考试差一分两分的例子，那么差一分和高一分的学生真的就是天壤之别吗？感知机也是如此：在超平面左侧距离原点0.001的点和右侧距离原点0.001的点输出就是+1和-1这天壤之别的差距真的合适吗？</p>
</div>
<div class="imageblock">
<div class="content">
<img src="images/luojihuigui3.png" alt="luojihuigui3">
</div>
</div>
<div class="paragraph">
<p>此外我们也知道机器学习中通常会对目标函数进行微分（求导）进而梯度下降，但我们看上面这张图。很明显\(x=0\)是跳跃间断点，因此\(sign\)是一条不光滑的函数，没有办法进行微分。咦？那我记得感知器用了梯度下降，它是怎么去梯度下降的？</p>
</div>
<div class="paragraph">
<p>我们回忆一下感知器的梯度下降方式，确实用了梯度下降，但是发现没有，它把sign的壳子去掉了，对sign内部进行梯度下降，相对于其他能直接微分的算法来说，感知器的这种方式确实有点不太好。</p>
</div>
<div class="paragraph">
<p>感知机算法：</p>
</div>
<div class="exampleblock">
<div class="content">
<div class="paragraph">
<p>输入：训练数据集\(T=(x_1,y_1),(x_2,y_2),\dots,(x_N,y_N),y_i \in \{-1,+1\},学习率 \eta (0 &lt; \eta &lt; 1)\)</p>
</div>
<div class="paragraph">
<p>输出：\(w,b\)；感知机模型\(f(x)=sign(wx+b)\)</p>
</div>
<div class="olist arabic">
<ol class="arabic">
<li>
<p>赋初值\(w_0,b_0\)</p>
</li>
<li>
<p>选取数据点\((x_i,y_i)\)</p>
</li>
<li>
<p>判断该数据点是否为当前模型的误分类点，即判断若\(y_i(wx_i+b) \le 0\)则更新：</p>
</li>
</ol>
</div>
<div class="stemblock">
<div class="content">
\[w \leftarrow w + \eta y_i x_i \\
b \leftarrow b + \eta y_i\]
</div>
</div>
<div class="olist arabic">
<ol class="arabic" start="4">
<li>
<p>转到2，直到训练集中没有误分类点。</p>
</li>
</ol>
</div>
</div>
</div>
<div class="paragraph">
<p>因此既然我们要对感知器进行优化，那么上文的这两个主要的缺陷咱们得想想法子看看能不能弥补。我们首先试试解决第一个问题：</p>
</div>
<div class="olist arabic">
<ol class="arabic">
<li>
<p>怎么解决极小距离差别带来的+1和-1的天壤之别？</p>
</li>
</ol>
</div>
<div class="paragraph">
<p>在逻辑斯蒂回归中大致思想与感知器相同，但在输出\(+1\)与\(-1\)之间存在一些差别。我们知道\(P(Y|X)\)，它表示在给定\(X\)条件下，\(Y\)发生的概率。逻辑回归也使用了同样的概念，它使用\(p(Y=-1|X)\)和\(p(Y=1|X)\)来表示该样本分别为\(-1\)或\(1\)的概率（实际上逻辑回归并非强制要求标签必须为1或-1，可以用任意标签来表示）。这样当再出现样本\(X_1\)距离为\(-0.001\)时，可能\(P(Y=1|X_1)=0.49,P(Y=0|X_1)=0.51\)，那么我们觉得\(X_1\)为0的概率更大一点，但我们同时也清楚程序可能并不太确定\(X_1\)一定为\(0\)。</p>
</div>
<div class="paragraph">
<p>使用概率作为输出结果使得样本在距离很小的差别下不再强制地输出\(+1\)和\(-1\)这两种天壤之别的结果，而是通过概率的方式告诉你结果可能是多少，同时也告诉你预测的不确信程度。这样子看起来让人比较安心一点不是吗？</p>
</div>
<div class="olist arabic">
<ol class="arabic">
<li>
<p>怎么让最终的预测式子可微呢？</p>
</li>
</ol>
</div>
<div class="paragraph">
<p>虽然无法微分并不会阻碍感知器的正常工作（事实上只是避开了sign），但对于很多场合都需要微分的机器学习来说，能找到一个可以微分的最终式子是很重要的。为了解决第一个问题我们提出了一种概率输出模型，那么感知器的\(sign\)也需要被随之替换为一种能输出概率大小的函数。具体函数在下文中详细讲解，其中值得高兴的是，我们找到的概率输出式子是平滑的，可微的，所以第二个问题也就迎刃而解了。</p>
</div>
</div>
<div class="sect3">
<h4 id="_逻辑回归模型">4.4.2. 逻辑回归模型</h4>
<div class="paragraph">
<p>与感知器一样，我们先不管内部怎么工作，最好能够得到一个函数\(f(x)\)，我们将样本\(x\)放进去以后，它告诉我们属于\(1\)的概率是多少。比如说\(f(y=1|x)=0.2,f(y=0|x)=0.8\)，我们就知道该样本的标签大概率是\(0\)。总结一下也就是分别比较两种类别的概率大小，概率大的那一方则作为预测类别进行输出。\(f\)函数的定义如下所示。</p>
</div>
<div class="paragraph">
<p>二项逻辑回归模型是如下的条件概率分布：</p>
</div>
<div class="stemblock">
<div class="content">
\[P(Y=1|x)=\frac{\exp{(wx+b)}}{1+\exp{(wx+b)}} \\
P(Y=0|x)=\frac{1}{1+\exp{(wx+b)}}\]
</div>
</div>
<div class="paragraph">
<p>这里，\(x \in R^n\)是输入，\(Y \in \{ 0,1 \}\)是输出，\(w \in R^n\)和\(b \in R\)是参数，\(w\)称为权值向量(weight)，\(b\)称为偏置(bias)，\(w \cdot x\)为\(w\)和\(x\)的内积。</p>
</div>
<div class="paragraph">
<p>下图就是\(f(w \cdot x+b)\)的图像，我们假设一下点\(X\)在超平面右边，那么距离应当为正，如果距离无穷大时，\(\exp{(w \cdot x + b)}\)无穷大，\(P(Y=1|x)=1\)，也就是概率为\(1\)，表示极其确信。如果\(w \cdot x+b\)是一个很接近\(0\)的数，\(\exp(w \cdot x + b)\)接近\(1\)，\(P(Y=1|x)=0.5\)，表示两边都有可能，不太确定。我们对于每一个样本点分别计算属于两类的概率，概率较大的一方作为预测的输出。</p>
</div>
<div class="imageblock">
<div class="content">
<img src="images/luojihuigui4.png" alt="luojihuigui4">
</div>
</div>
<div class="admonitionblock note">
<table>
<tr>
<td class="icon">
<i class="fa icon-note" title="Note"></i>
</td>
<td class="content">
其实这里的函数\(\phi (z)=\frac{1}{1+e^{-z}}\)，有一个很有名的名字叫做Sigmoid Function，或者叫压缩函数，在神经网络里面经常用到。
</td>
</tr>
</table>
</div>
</div>
<div class="sect3">
<h4 id="_损失函数_3">4.4.3. 损失函数</h4>
<div class="paragraph">
<p>好了，所要用的几个函数我们都有了，接下来要做的就是根据给定的训练集，把参数\(w\)给求出来了。要找参数\(w\)，首先就是得把损失函数（cost function）给定义出来，也就是目标函数。</p>
</div>
<div class="paragraph">
<p>我们第一个想到的自然是模仿线性回归的做法，利用误差平方和来当代价函数。</p>
</div>
<div class="stemblock">
<div class="content">
\[L(w,b)=\frac{1}{2} \sum_{i=1}^N (\phi (w \cdot x_i + b)-y_i)^2\]
</div>
</div>
<div class="paragraph">
<p>其中\((x_i,y_i)\)为样本点。\(y_i\)为真实值，\(\phi (w \cdot x_i + b)\)为预测值。</p>
</div>
<div class="paragraph">
<p>我们发现，逻辑回归的平方损失函数是一个非凸函数，这就意味着损失函数有着许多的局部极小值，这不利于我们求解最小值。</p>
</div>
<div class="imageblock">
<div class="content">
<img src="images/luojihuigui5.png" alt="luojihuigui5">
</div>
</div>
<div class="paragraph">
<p>所以我们这里使用<strong>对数损失函数</strong>。</p>
</div>
<div class="ulist">
<ul>
<li>
<p>对数损失函数</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[L(Y,P(Y \vert X))=-logP(Y \vert X)\]
</div>
</div>
<div class="paragraph">
<p>也就是说</p>
</div>
<div class="stemblock">
<div class="content">
\[L(w,b)=
\begin{cases}
  -\log \phi (wx+b) &amp; if y = 1 \\
  -\log (1-\phi (wx+b)) &amp; if y = 0
\end{cases}\]
</div>
</div>
<div class="imageblock">
<div class="content">
<img src="images/luojihuigui6.png" alt="luojihuigui6">
</div>
</div>
<div class="paragraph">
<p>也就是说</p>
</div>
<div class="paragraph">
<p>当样本值\(y=1\)，而预测值\(\phi (wx+b) = \frac{\exp{(wx+b)}}{1+\exp{(wx+b)}}\)接近于0时，也就是说完全预测错误，那么惩罚将是很大的，是无穷大。\(y=0\)时同理。</p>
</div>
<div class="paragraph">
<p>我们可以得出</p>
</div>
<div class="stemblock">
<div class="content">
\[L(w,b)=
\begin{cases}
  -\log \phi (wx+b) &amp; if y = 1 \\
  -\log (1-\phi (wx+b)) &amp; if y = 0
\end{cases} \\
\Downarrow \\
L(w,b) = -y \log \phi (wx+b) - (1 - y) \log (1 - \phi (wx+b)) \\
\Downarrow \\
L(w,b) = \frac{1}{m} \sum_{i=1}^m (-y_i \log \phi (wx_i+b) - (1 - y_i) \log (1 - \phi (wx_i+b)))\]
</div>
</div>
</div>
<div class="sect3">
<h4 id="_梯度下降">4.4.4. 梯度下降</h4>
<div class="admonitionblock note">
<table>
<tr>
<td class="icon">
<i class="fa icon-note" title="Note"></i>
</td>
<td class="content">
Sigmoid Function有一个很好的性质：
</td>
</tr>
</table>
</div>
<div class="stemblock">
<div class="content">
\[\phi ' (z)=\phi (z)(1-\phi (z))\]
</div>
</div>
<div class="paragraph">
<p>求偏导数的过程</p>
</div>
<div class="stemblock">
<div class="content">
\[\frac{\partial L(w,b)}{\partial w}
=
\sum_{i=1}^m (y_ix_i-\frac{x_i \exp (wx_i)}{1+ \exp (wx_i)})\]
</div>
</div>
</div>
</div>
<div class="sect2">
<h3 id="_knn">4.5. KNN</h3>
<div class="ulist">
<ul>
<li>
<p>最简单最初级的分类器，就是将全部的训练数据所对应的类别都记录下来，当测试对象的属性和某个训练对象的属性完全匹配时，便可以对其进行分类</p>
</li>
<li>
<p>K近邻（k-nearest neighbour, KNN）是一种基本分类方法，通过测量不同特征值之间的距离进行分类。它的思路是：如果一个样本在特征空间中的k个最相似（即特征空间中最邻近）的样本中的大多数属于某一个类别，则该样本也属于这个类别，其中K通常是不大于20的整数</p>
</li>
<li>
<p>KNN算法中，所选择的邻居都是已经正确分类的对象</p>
</li>
</ul>
</div>
<div class="sect3">
<h4 id="_knn例子">4.5.1. KNN例子</h4>
<div class="imageblock">
<div class="content">
<img src="images/knn.png" alt="knn">
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>绿色圆要被决定赋予哪个类，是红色三角形还是蓝色四方形？</p>
</li>
<li>
<p>如果K=3，由于红色三角形所占比例为2/3，绿色圆将被赋予红色三角形那个类，如果K=5，由于蓝色四方形比例为3/5，因此绿色圆被赋予蓝色四方形类</p>
</li>
<li>
<p>KNN算法的结果很大程度取决于K的选择</p>
</li>
</ul>
</div>
</div>
<div class="sect3">
<h4 id="_knn距离计算">4.5.2. KNN距离计算</h4>
<div class="paragraph">
<p>KNN中，通过计算对象间距离来作为各个对象之间的非相似性指标，避免了对象之间的匹配问题，在这里距离一般使用欧氏距离或曼哈顿距离：</p>
</div>
<div class="ulist">
<ul>
<li>
<p>欧式距离</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[d(x,y) = \sqrt {\sum_{k=1}^n (x_k - y_k)^2}\]
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>曼哈顿距离</p>
</li>
</ul>
</div>
<div class="stemblock">
<div class="content">
\[d(x,y) = \sqrt {\sum_{k=1}^n | x_k - y_k |}\]
</div>
</div>
</div>
<div class="sect3">
<h4 id="_knn算法">4.5.3. KNN算法</h4>
<div class="exampleblock">
<div class="content">
<div class="paragraph">
<p>在训练集中数据和标签已知的情况下，输入测试数据，将测试数据的特征与训练集中对应的特征进行相互比较，找到训练集中与之最为相似的前K个数据，则该测试数据对应的类别就是K个数据中出现次数最多的那个分类，其算法的描述为：</p>
</div>
<div class="olist arabic">
<ol class="arabic">
<li>
<p>计算测试数据与各个训练数据之间的距离；</p>
</li>
<li>
<p>按照距离的递增关系进行排序；</p>
</li>
<li>
<p>选取距离最小的K个点；</p>
</li>
<li>
<p>确定前K个点所在类别的出现频率；</p>
</li>
<li>
<p>返回前K个点中出现频率最高的类别作为测试数据的预测分类。</p>
</li>
</ol>
</div>
</div>
</div>
</div>
</div>
<div class="sect2">
<h3 id="_决策树">4.6. 决策树</h3>

</div>
</div>
</div>
<div class="sect1">
<h2 id="_推荐系统项目实战">5. 推荐系统项目实战</h2>
<div class="sectionbody">
<div class="sect2">
<h3 id="_项目体系架构设计">5.1. 项目体系架构设计</h3>
<div class="sect3">
<h4 id="_项目系统架构">5.1.1. 项目系统架构</h4>
<div class="paragraph">
<p>项目以推荐系统建设领域知名的经过修改过的中文亚马逊电商数据集作为依托，以某电商网站真实业务数据架构为基础，构建了符合教学体系的一体化的电商推荐系统，包含了离线推荐与实时推荐体系，综合利用了协同过滤算法以及基于内容的推荐方法来提供混合推荐。提供了从前端应用、后台服务、算法设计实现、平台部署等多方位的闭环的业务实现。</p>
</div>
<div class="paragraph">
<p>电商推荐系统架构图</p>
</div>
<div class="imageblock">
<div class="content">
<img src="images/架构图.png" alt="架构图">
</div>
</div>
<div class="paragraph">
<p>架构图代码，使用mermaid渲染出架构图</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code><span></span>graph TD
A[前端可视化, Angular] --&gt; B[后端业务逻辑, Spring]
B --&gt; C[业务数据库]
C --&gt; D[持久化: MongoDB]
C --&gt; E[缓存: Redis]
D --&gt; F[离线推荐服务, Spark MLlib]
D --&gt; G[离线统计服务, Spark SQL]
B --&gt; H[日志采集服务, Flume]
H --&gt; I[消息队列, Kafka]
I --&gt; J[实时推荐, Spark Streaming]
E --&gt; J</code></pre>
</div>
</div>
<div class="ulist">
<ul>
<li>
<p>用户可视化：主要负责实现和用户的交互以及业务数据的展示，主体采用AngularJS2进行实现，部署在Apache服务上。</p>
</li>
<li>
<p>综合业务服务：主要实现JavaEE层面整体的业务逻辑，通过Spring进行构建，对接业务需求。部署在Tomcat上。</p>
</li>
</ul>
</div>
</div>
<div class="sect3">
<h4 id="_数据存储部分">5.1.2. 数据存储部分</h4>
<div class="ulist">
<ul>
<li>
<p>业务数据库：项目采用广泛应用的文档数据库MongoDB作为主数据库，主要负责平台业务逻辑数据的存储。</p>
</li>
<li>
<p>缓存数据库：项目采用Redis作为缓存数据库，主要用来支撑实时推荐系统部分对于数据的高速获取需求。</p>
</li>
</ul>
</div>
<div class="sect4">
<h5 id="_离线推荐部分">离线推荐部分</h5>
<div class="ulist">
<ul>
<li>
<p>离线统计服务：批处理统计性业务采用Spark Core + Spark SQL进行实现，实现对指标类数据的统计任务。</p>
</li>
<li>
<p>离线推荐服务：离线推荐业务采用Spark Core + Spark MLlib进行实现，采用ALS算法进行实现。</p>
</li>
<li>
<p>基于内容的推荐：采用TF-IDF算法提取UGC标签的关键词，计算商品之间的余弦相似度。</p>
</li>
<li>
<p>基于物品的协同过滤推荐：实现传统的Item-CF算法。</p>
</li>
</ul>
</div>
</div>
<div class="sect4">
<h5 id="_实时推荐部分">实时推荐部分</h5>
<div class="ulist">
<ul>
<li>
<p>日志采集服务：通过利用Flume-ng对业务平台中用户对于电影的一次评分行为进行采集，实时发送到Kafka集群。</p>
</li>
<li>
<p>消息缓冲服务：项目采用Kafka作为流式数据的缓存组件，接受来自Flume的数据采集请求。并将数据推送到项目的实时推荐系统部分。</p>
</li>
<li>
<p>实时推荐服务：项目采用Spark Streaming作为实时推荐系统，通过接收Kafka中缓存的数据，通过设计的推荐算法实现对实时推荐的数据处理，并将结构合并更新到MongoDB数据库。</p>
</li>
</ul>
</div>
</div>
</div>
</div>
<div class="sect2">
<h3 id="_项目数据流程">5.2. 项目数据流程</h3>
<div class="paragraph">
<p>电商推荐系统数据流图</p>
</div>
<div class="paragraph">
<p><strong>略</strong></p>
</div>
<div class="sect3">
<h4 id="_系统初始化部分">5.2.1. 系统初始化部分</h4>
<div class="olist arabic">
<ol class="arabic">
<li>
<p>通过Spark SQL将系统初始化数据加载到MongoDB中。
【离线推荐部分】</p>
</li>
<li>
<p>离线统计服务从MongoDB中加载数据，将【商品平均评分统计】、【商品评分个数统计】、【最近商品评分个数统计】三个统计算法进行运行实现，并将计算结果回写到MongoDB中；离线推荐服务从MongoDB中加载数据，通过ALS算法分别将【用户推荐结果矩阵】、【商品相似度矩阵】回写到MongoDB中。</p>
</li>
</ol>
</div>
</div>
<div class="sect3">
<h4 id="_实时推荐部分_2">5.2.2. 实时推荐部分</h4>
<div class="olist arabic">
<ol class="arabic">
<li>
<p>Flume从综合业务服务的运行日志中读取日志更新，并将更新的日志实时推送到Kafka中；Kafka在收到这些日志之后，通过kafkaStream程序对获取的日志信息进行过滤处理，获取用户评分数据流【userId|productId|score|timestamp】，并发送到另外一个Kafka队列；Spark Streaming监听Kafka队列，实时获取Kafka过滤出来的用户评分数据流，融合存储在Redis中的用户最近评分队列数据，提交给实时推荐算法，完成对用户新的推荐结果计算；计算完成之后，将新的推荐结构和MongDB数据库中的推荐结果进行合并。</p>
</li>
</ol>
</div>
</div>
<div class="sect3">
<h4 id="_业务系统部分">5.2.3. 业务系统部分</h4>
<div class="olist arabic">
<ol class="arabic">
<li>
<p>推荐结果展示部分，从MongoDB中将离线推荐结果、实时推荐结果、内容推荐结果进行混合，综合给出相对应的数据。</p>
</li>
<li>
<p>商品信息查询服务通过对接MongoDB实现对商品信息的查询操作。</p>
</li>
<li>
<p>商品评分部分，获取用户通过UI给出的评分动作，后台服务进行数据库记录后，一方面将数据推动到Redis群中，另一方面，通过预设的日志框架输出到Tomcat中的日志中。</p>
</li>
<li>
<p>商品标签部分，项目提供用户对电影打标签服务。</p>
</li>
</ol>
</div>
</div>
</div>
<div class="sect2">
<h3 id="_数据模型">5.3. 数据模型</h3>
<div class="olist arabic">
<ol class="arabic">
<li>
<p>商品数据表</p>
</li>
</ol>
</div>
<table class="tableblock frame-all grid-all stretch">
<colgroup>
<col style="width: 25%;">
<col style="width: 25%;">
<col style="width: 25%;">
<col style="width: 25%;">
</colgroup>
<thead>
<tr>
<th class="tableblock halign-left valign-top">字段名</th>
<th class="tableblock halign-left valign-top">字段类型</th>
<th class="tableblock halign-left valign-top">字段描述</th>
<th class="tableblock halign-left valign-top">字段备注</th>
</tr>
</thead>
<tbody>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">productId</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Int</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">商品ID</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">name</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">String</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">商品名字</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">categories</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">String</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">商品分类</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">imageUrl</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">String</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">图片url</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">tags</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">String</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">UGC标签</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
</tbody>
</table>
<div class="olist arabic">
<ol class="arabic">
<li>
<p>评分数据表</p>
</li>
</ol>
</div>
<table class="tableblock frame-all grid-all stretch">
<colgroup>
<col style="width: 25%;">
<col style="width: 25%;">
<col style="width: 25%;">
<col style="width: 25%;">
</colgroup>
<thead>
<tr>
<th class="tableblock halign-left valign-top">字段名</th>
<th class="tableblock halign-left valign-top">字段类型</th>
<th class="tableblock halign-left valign-top">字段描述</th>
<th class="tableblock halign-left valign-top">字段备注</th>
</tr>
</thead>
<tbody>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">userid</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Int</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">用户ID</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">productId</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Int</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">商品ID</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">score</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Double</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">商品分值</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">timestamp</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Long</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">评分时间</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
</tbody>
</table>
<div class="olist arabic">
<ol class="arabic">
<li>
<p>用户表</p>
</li>
</ol>
</div>
<table class="tableblock frame-all grid-all stretch">
<colgroup>
<col style="width: 25%;">
<col style="width: 25%;">
<col style="width: 25%;">
<col style="width: 25%;">
</colgroup>
<thead>
<tr>
<th class="tableblock halign-left valign-top">字段名</th>
<th class="tableblock halign-left valign-top">字段类型</th>
<th class="tableblock halign-left valign-top">字段描述</th>
<th class="tableblock halign-left valign-top">字段备注</th>
</tr>
</thead>
<tbody>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">productId</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Int</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">用户的ID</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">username</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">String</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">用户名</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">password</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">String</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">用户密码</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">first</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">boolean</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">用于是否第一次登录</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">timestamp</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Long</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">用户创建的时间</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
</tbody>
</table>
</div>
<div class="sect2">
<h3 id="_工具环境搭建">5.4. 工具环境搭建</h3>
<div class="paragraph">
<p>我们的项目中用到了多种工具进行数据的存储、计算、采集和传输，本章主要简单介绍设计的工具环境搭建。</p>
</div>
<div class="paragraph">
<p>如果机器的配置不足，推荐只采用一台虚拟机进行配置，而非完全分布式，将该虚拟机CPU的内存设置的尽可能大，推荐为CPU &gt; 4、MEM &gt; 4GB。</p>
</div>
<div class="sect3">
<h4 id="_mongodb单节点环境配置">5.4.1. MongoDB（单节点）环境配置</h4>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="sh"><span></span>// 通过WGET下载Linux版本的MongoDB
<span class="tok-o">[</span>bigdata@linux ~<span class="tok-o">]</span>$ wget https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel62-3.4.3.tgz
// 将压缩包解压到指定目录
<span class="tok-o">[</span>bigdata@linux backup<span class="tok-o">]</span>$ tar -xf mongodb-linux-x86_64-rhel62-3.4.3.tgz -C ~/
// 将解压后的文件移动到最终的安装目录
<span class="tok-o">[</span>bigdata@linux ~<span class="tok-o">]</span>$ mv mongodb-linux-x86_64-rhel62-3.4.3/ /usr/local/mongodb
// 在安装目录下创建data文件夹用于存放数据和日志
<span class="tok-o">[</span>bigdata@linux mongodb<span class="tok-o">]</span>$ mkdir /usr/local/mongodb/data/
// 在data文件夹下创建db文件夹，用于存放数据
<span class="tok-o">[</span>bigdata@linux mongodb<span class="tok-o">]</span>$ mkdir /usr/local/mongodb/data/db/
// 在data文件夹下创建logs文件夹，用于存放日志
<span class="tok-o">[</span>bigdata@linux mongodb<span class="tok-o">]</span>$ mkdir /usr/local/mongodb/data/logs/
// 在logs文件夹下创建log文件
<span class="tok-o">[</span>bigdata@linux mongodb<span class="tok-o">]</span>$ touch /usr/local/mongodb/data/logs/ mongodb.log

// 在data文件夹下创建mongodb.conf配置文件
<span class="tok-o">[</span>bigdata@linux mongodb<span class="tok-o">]</span>$ touch /usr/local/mongodb/data/mongodb.conf
// 在mongodb.conf文件中输入如下内容
<span class="tok-o">[</span>bigdata@linux mongodb<span class="tok-o">]</span>$ vim ./data/mongodb.conf</code></pre>
</div>
</div>
<div class="paragraph">
<p>配置文件内容：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code><span></span>#端口号port = 27017
#数据目录
dbpath = /usr/local/mongodb/data/db
#日志目录
logpath = /usr/local/mongodb/data/logs/mongodb.log
#设置后台运行
fork = true
#日志输出方式
logappend = true</code></pre>
</div>
</div>
<div class="paragraph">
<p>完成MongoDB的安装后，启动MongoDB服务器：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="sh"><span></span>// 启动MongoDB服务器
<span class="tok-o">[</span>bigdata@linux mongodb<span class="tok-o">]</span>$ sudo /usr/local/mongodb/bin/mongod -config /usr/local/mongodb/data/mongodb.conf
// 访问MongoDB服务器
<span class="tok-o">[</span>bigdata@linux mongodb<span class="tok-o">]</span>$ /usr/local/mongodb/bin/mongo
// 停止MongoDB服务器
<span class="tok-o">[</span>bigdata@linux mongodb<span class="tok-o">]</span>$ sudo /usr/local/mongodb/bin/mongod -shutdown -config /usr/local/mongodb/data/mongodb.conf</code></pre>
</div>
</div>
</div>
<div class="sect3">
<h4 id="_redis单节点环境配置">5.4.2. Redis（单节点）环境配置</h4>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="sh"><span></span>// 通过WGET下载REDIS的源码
<span class="tok-o">[</span>bigdata@linux ~<span class="tok-o">]</span><span class="tok-nv">$wget</span> http://download.redis.io/releases/redis-4.0.2.tar.gz
// 将源代码解压到安装目录
<span class="tok-o">[</span>bigdata@linux ~<span class="tok-o">]</span>$ tar -xf redis-4.0.2.tar.gz -C ~/
// 进入Redis源代码目录，编译安装
<span class="tok-o">[</span>bigdata@linux ~<span class="tok-o">]</span>$ <span class="tok-nb">cd</span> redis-4.0.2/
// 安装GCC
<span class="tok-o">[</span>bigdata@linux ~<span class="tok-o">]</span>$ sudo yum install gcc
// 编译源代码
<span class="tok-o">[</span>bigdata@linux redis-4.0.2<span class="tok-o">]</span>$ make <span class="tok-nv">MALLOC</span><span class="tok-o">=</span>libc
// 编译安装
<span class="tok-o">[</span>bigdata@linux redis-4.0.2<span class="tok-o">]</span>$ sudo make install
// 创建配置文件
<span class="tok-o">[</span>bigdata@linux redis-4.0.2<span class="tok-o">]</span>$ sudo cp ~/redis-4.0.2/redis.conf /etc/
// 修改配置文件中以下内容
<span class="tok-o">[</span>bigdata@linux redis-4.0.2<span class="tok-o">]</span>$ sudo vim /etc/redis.conf
daemonize yes   <span class="tok-c1">#37行  #是否以后台daemon方式运行，默认不是后台运行</span>
pidfile /var/run/redis/redis.pid   <span class="tok-c1">#41行  #redis的PID文件路径（可选）</span>
<span class="tok-nb">bind</span> <span class="tok-m">0</span>.0.0.0    <span class="tok-c1">#64行  #绑定主机IP，默认值为127.0.0.1，我们是跨机器运行，所以需要更改</span>
logfile /var/log/redis/redis.log   <span class="tok-c1">#104行  #定义log文件位置，模式log信息定向到stdout，输出到/dev/null（可选）</span>
dir “/usr/local/rdbfile”  <span class="tok-c1">#188行  #本地数据库存放路径，默认为./，编译安装默认存在在/usr/local/bin下（可选）</span>
在安装完Redis之后，启动Redis
// 启动Redis服务器
<span class="tok-o">[</span>bigdata@linux redis-4.0.2<span class="tok-o">]</span>$ redis-server /etc/redis.conf
// 连接Redis服务器
<span class="tok-o">[</span>bigdata@linux redis-4.0.2<span class="tok-o">]</span>$ redis-cli
// 停止Redis服务器
<span class="tok-o">[</span>bigdata@linux redis-4.0.2<span class="tok-o">]</span>$ redis-cli shutdown</code></pre>
</div>
</div>
<div class="paragraph">
<p>在安装完Redis之后，启动Redis</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="sh"><span></span>// 启动Redis服务器
<span class="tok-o">[</span>bigdata@linux redis-4.0.2<span class="tok-o">]</span>$ redis-server /etc/redis.conf
// 连接Redis服务器
<span class="tok-o">[</span>bigdata@linux redis-4.0.2<span class="tok-o">]</span>$ redis-cli
// 停止Redis服务器
<span class="tok-o">[</span>bigdata@linux redis-4.0.2<span class="tok-o">]</span>$ redis-cli shutdown</code></pre>
</div>
</div>
</div>
<div class="sect3">
<h4 id="_spark单节点环境配置">5.4.3. Spark（单节点）环境配置</h4>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="sh"><span></span>// 通过wget下载zookeeper安装包
<span class="tok-o">[</span>bigdata@linux ~<span class="tok-o">]</span>$ wget https://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.7.tgz
// 将spark解压到安装目录
<span class="tok-o">[</span>bigdata@linux ~<span class="tok-o">]</span>$ tar –xf spark-2.1.1-bin-hadoop2.7.tgz –C ./cluster
// 进入spark安装目录
<span class="tok-o">[</span>bigdata@linux cluster<span class="tok-o">]</span>$ <span class="tok-nb">cd</span> spark-2.1.1-bin-hadoop2.7/
// 复制slave配置文件
<span class="tok-o">[</span>bigdata@linux spark-2.1.1-bin-hadoop2.7<span class="tok-o">]</span>$ cp ./conf/slaves.template ./conf/slaves
// 修改slave配置文件
<span class="tok-o">[</span>bigdata@linux spark-2.1.1-bin-hadoop2.7<span class="tok-o">]</span>$ vim ./conf/slaves
linux  <span class="tok-c1">#在文件最后将本机主机名进行添加</span>
// 复制Spark-Env配置文件
<span class="tok-o">[</span>bigdata@linux spark-2.1.1-bin-hadoop2.7<span class="tok-o">]</span>$ cp ./conf/spark-env.sh.template ./conf/spark-env.sh
<span class="tok-nv">SPARK_MASTER_HOST</span><span class="tok-o">=</span>linux       <span class="tok-c1">#添加spark master的主机名</span>
<span class="tok-nv">SPARK_MASTER_PORT</span><span class="tok-o">=</span><span class="tok-m">7077</span>        <span class="tok-c1">#添加spark master的端口号</span></code></pre>
</div>
</div>
<div class="paragraph">
<p>安装完成之后，启动Spark</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="sh"><span></span>// 启动Spark集群
<span class="tok-o">[</span>bigdata@linux spark-2.1.1-bin-hadoop2.7<span class="tok-o">]</span>$ sbin/start-all.sh
// 访问Spark集群，浏览器访问http://linux:8080

// 关闭Spark集群
<span class="tok-o">[</span>bigdata@linux spark-2.1.1-bin-hadoop2.7<span class="tok-o">]</span>$ sbin/stop-all.sh</code></pre>
</div>
</div>
</div>
<div class="sect3">
<h4 id="_zookeeper单节点环境配置">5.4.4. Zookeeper（单节点）环境配置</h4>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="sh"><span></span>// 通过wget下载zookeeper安装包
<span class="tok-o">[</span>bigdata@linux ~<span class="tok-o">]</span>$ wget http://mirror.bit.edu.cn/apache/zookeeper/zookeeper-3.4.10/zookeeper-3.4.10.tar.gz
// 将zookeeper解压到安装目录
<span class="tok-o">[</span>bigdata@linux ~<span class="tok-o">]</span>$ tar –xf zookeeper-3.4.10.tar.gz –C ./cluster
// 进入zookeeper安装目录
<span class="tok-o">[</span>bigdata@linux cluster<span class="tok-o">]</span>$ <span class="tok-nb">cd</span> zookeeper-3.4.10/
// 创建data数据目录
<span class="tok-o">[</span>bigdata@linux zookeeper-3.4.10<span class="tok-o">]</span>$ mkdir data/
// 复制zookeeper配置文件
<span class="tok-o">[</span>bigdata@linux zookeeper-3.4.10<span class="tok-o">]</span>$ cp ./conf/zoo_sample.cfg ./conf/zoo.cfg
// 修改zookeeper配置文件
<span class="tok-o">[</span>bigdata@linux zookeeper-3.4.10<span class="tok-o">]</span>$ vim conf/zoo.cfg
<span class="tok-nv">dataDir</span><span class="tok-o">=</span>/home/bigdata/cluster/zookeeper-3.4.10/data  <span class="tok-c1">#将数据目录地址修改为创建的目录</span>
// 启动Zookeeper服务
<span class="tok-o">[</span>bigdata@linux zookeeper-3.4.10<span class="tok-o">]</span>$ bin/zkServer.sh start
// 查看Zookeeper服务状态
<span class="tok-o">[</span>bigdata@linux zookeeper-3.4.10<span class="tok-o">]</span>$ bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /home/bigdata/cluster/zookeeper-3.4.10/bin/../conf/zoo.cfg
Mode: standalone
// 关闭Zookeeper服务
<span class="tok-o">[</span>bigdata@linux zookeeper-3.4.10<span class="tok-o">]</span>$ bin/zkServer.sh stop</code></pre>
</div>
</div>
</div>
<div class="sect3">
<h4 id="_flume_ng单节点环境配置">5.4.5. Flume-ng（单节点）环境配置</h4>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="sh"><span></span>// 通过wget下载zookeeper安装包
<span class="tok-o">[</span>bigdata@linux ~<span class="tok-o">]</span>$ wget http://www.apache.org/dyn/closer.lua/flume/1.8.0/apache-flume-1.8.0-bin.tar.gz
// 将zookeeper解压到安装目录
<span class="tok-o">[</span>bigdata@linux ~<span class="tok-o">]</span>$ tar –xf apache-flume-1.8.0-bin.tar.gz –C ./cluster
// 等待项目部署时使用</code></pre>
</div>
</div>
</div>
<div class="sect3">
<h4 id="_kafka单节点环境配置">5.4.6. Kafka（单节点）环境配置</h4>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="sh"><span></span>// 通过wget下载zookeeper安装包
<span class="tok-o">[</span>bigdata@linux ~<span class="tok-o">]</span>$ wget http://mirrors.tuna.tsinghua.edu.cn/apache/kafka/0.10.2.1/kafka_2.11-0.10.2.1.tgz
// 将kafka解压到安装目录
<span class="tok-o">[</span>bigdata@linux ~<span class="tok-o">]</span>$ tar –xf kafka_2.12-0.10.2.1.tgz –C ./cluster
// 进入kafka安装目录
<span class="tok-o">[</span>bigdata@linux cluster<span class="tok-o">]</span>$ <span class="tok-nb">cd</span> kafka_2.12-0.10.2.1/
// 修改kafka配置文件
<span class="tok-o">[</span>bigdata@linux kafka_2.12-0.10.2.1<span class="tok-o">]</span>$ vim config/server.properties
host.name<span class="tok-o">=</span>linux                  <span class="tok-c1">#修改主机名</span>
<span class="tok-nv">port</span><span class="tok-o">=</span><span class="tok-m">9092</span>                         <span class="tok-c1">#修改服务端口号</span>
zookeeper.connect<span class="tok-o">=</span>linux:2181     <span class="tok-c1">#修改Zookeeper服务器地址</span>
// 启动kafka服务 !!! 启动之前需要启动Zookeeper服务
<span class="tok-o">[</span>bigdata@linux kafka_2.12-0.10.2.1<span class="tok-o">]</span>$ bin/kafka-server-start.sh -daemon ./config/server.properties
// 关闭kafka服务
<span class="tok-o">[</span>bigdata@linux kafka_2.12-0.10.2.1<span class="tok-o">]</span>$ bin/kafka-server-stop.sh
// 创建topic
<span class="tok-o">[</span>bigdata@linux kafka_2.12-0.10.2.1<span class="tok-o">]</span>$ bin/kafka-topics.sh --create --zookeeper linux:2181 --replication-factor <span class="tok-m">1</span> --partitions <span class="tok-m">1</span> --topic recommender
// kafka-console-producer
<span class="tok-o">[</span>bigdata@linux kafka_2.12-0.10.2.1<span class="tok-o">]</span>$ bin/kafka-console-producer.sh --broker-list linux:9092 --topic recommender
// kafka-console-consumer
<span class="tok-o">[</span>bigdata@linux kafka_2.12-0.10.2.1<span class="tok-o">]</span>$ bin/kafka-console-consumer.sh --bootstrap-server linux:9092 --topic recommender</code></pre>
</div>
</div>
</div>
</div>
<div class="sect2">
<h3 id="_创建项目并初始化业务数据">5.5. 创建项目并初始化业务数据</h3>
<div class="paragraph">
<p>我们的项目主体用Scala编写，采用IDEA作为开发环境进行项目编写，采用maven作为项目构建和管理工具。</p>
</div>
<div class="sect3">
<h4 id="_在idea中创建maven项目">5.5.1. 在IDEA中创建maven项目</h4>
<div class="paragraph">
<p>打开IDEA，创建一个maven项目，命名为ECommerceRecommendSystem。为了方便后期的联调，我们会把业务系统的代码也添加进来，所以我们可以以ECommerceRecommendSystem作为父项目，并在其下建一个名为recommender的子项目，然后再在下面搭建多个子项目用于提供不同的推荐服务。</p>
</div>
</div>
<div class="sect3">
<h4 id="_项目框架搭建">5.5.2. 项目框架搭建</h4>
<div class="paragraph">
<p>在ECommerceRecommendSystem的pom.xml文件中加入元素`&lt;packaging&gt;pom&lt;/packaging&gt;`，然后新建一个maven module作为子项目，命名为recommender。同样的，再以recommender为父项目，在它的pom.xml中加入`&lt;packing&gt;pom&lt;/packaging&gt;`，然后新建一个maven module作为子项目。我们的第一步是初始化业务数据，所以子项目命名为DataLoader。</p>
</div>
<div class="paragraph">
<p>父项目只是为了规范化项目结构，方便依赖管理，本身是不需要代码实现的，所以ECommerceRecommendSystem和recommender下的src文件夹都可以删掉。</p>
</div>
<div class="paragraph">
<p>目前的整体项目框架如下：</p>
</div>
<div class="paragraph">
<p><strong>略</strong></p>
</div>
</div>
<div class="sect3">
<h4 id="_声明项目中工具的版本信息">5.5.3. 声明项目中工具的版本信息</h4>
<div class="paragraph">
<p>我们整个项目需要用到多个工具，它们的不同版本可能会对程序运行造成影响，所以应该在最外层的ECommerceRecommendSystem中声明所有子项目共用的版本信息。
在pom.xml中加入以下配置：</p>
</div>
<div class="paragraph">
<p><code>ECommerceRecommendSystem/pom.xml</code></p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="xml"><span></span>    <span class="tok-nt">&lt;properties&gt;</span>
        <span class="tok-nt">&lt;mysql.version&gt;</span>6.0.5<span class="tok-nt">&lt;/mysql.version&gt;</span>
        <span class="tok-nt">&lt;shiro.version&gt;</span>1.3.2<span class="tok-nt">&lt;/shiro.version&gt;</span>
        <span class="tok-nt">&lt;spring.version&gt;</span>4.3.6.RELEASE<span class="tok-nt">&lt;/spring.version&gt;</span>
        <span class="tok-nt">&lt;spring.data.jpa.version&gt;</span>1.11.0.RELEASE<span class="tok-nt">&lt;/spring.data.jpa.version&gt;</span>
        <span class="tok-nt">&lt;log4j.version&gt;</span>1.2.17<span class="tok-nt">&lt;/log4j.version&gt;</span>
        <span class="tok-nt">&lt;quartz.version&gt;</span>2.2.3<span class="tok-nt">&lt;/quartz.version&gt;</span>
        <span class="tok-nt">&lt;slf4j.version&gt;</span>1.7.22<span class="tok-nt">&lt;/slf4j.version&gt;</span>
        <span class="tok-nt">&lt;hibernate.version&gt;</span>5.2.6.Final<span class="tok-nt">&lt;/hibernate.version&gt;</span>
        <span class="tok-nt">&lt;camel.version&gt;</span>2.18.2<span class="tok-nt">&lt;/camel.version&gt;</span>
        <span class="tok-nt">&lt;freemarker.version&gt;</span>2.3.23<span class="tok-nt">&lt;/freemarker.version&gt;</span>
        <span class="tok-nt">&lt;config.version&gt;</span>1.10<span class="tok-nt">&lt;/config.version&gt;</span>
        <span class="tok-nt">&lt;jackson.version&gt;</span>2.8.6<span class="tok-nt">&lt;/jackson.version&gt;</span>
        <span class="tok-nt">&lt;servlet.version&gt;</span>3.0.1<span class="tok-nt">&lt;/servlet.version&gt;</span>
        <span class="tok-nt">&lt;net.sf.json.version&gt;</span>2.4<span class="tok-nt">&lt;/net.sf.json.version&gt;</span>
        <span class="tok-nt">&lt;activemq.version&gt;</span>5.14.3<span class="tok-nt">&lt;/activemq.version&gt;</span>
        <span class="tok-nt">&lt;spark.version&gt;</span>2.1.1<span class="tok-nt">&lt;/spark.version&gt;</span>
        <span class="tok-nt">&lt;scala.version&gt;</span>2.11.8<span class="tok-nt">&lt;/scala.version&gt;</span>
        <span class="tok-nt">&lt;hadoop.version&gt;</span>2.7.3<span class="tok-nt">&lt;/hadoop.version&gt;</span>
        <span class="tok-nt">&lt;mongodb-spark.version&gt;</span>2.0.0<span class="tok-nt">&lt;/mongodb-spark.version&gt;</span>
        <span class="tok-nt">&lt;casbah.version&gt;</span>3.1.1<span class="tok-nt">&lt;/casbah.version&gt;</span>
        <span class="tok-nt">&lt;elasticsearch-spark.version&gt;</span>5.6.2<span class="tok-nt">&lt;/elasticsearch-spark.version&gt;</span>
        <span class="tok-nt">&lt;elasticsearch.version&gt;</span>5.6.2<span class="tok-nt">&lt;/elasticsearch.version&gt;</span>
        <span class="tok-nt">&lt;jblas.version&gt;</span>1.2.1<span class="tok-nt">&lt;/jblas.version&gt;</span>
    <span class="tok-nt">&lt;/properties&gt;</span></code></pre>
</div>
</div>
</div>
<div class="sect3">
<h4 id="_添加项目依赖">5.5.4. 添加项目依赖</h4>
<div class="paragraph">
<p>首先，对于整个项目而言，应该有同样的日志管理，我们在ECommerceRecommendSystem中引入公有依赖：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="xml"><span></span>    <span class="tok-nt">&lt;dependencies&gt;</span>
        <span class="tok-c">&lt;!-- Logging --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.slf4j<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>jcl-over-slf4j<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${slf4j.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.slf4j<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>slf4j-api<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${slf4j.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.slf4j<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>slf4j-log4j12<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${slf4j.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>log4j<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>log4j<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${log4j.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-c">&lt;!-- Logging End --&gt;</span>
    <span class="tok-nt">&lt;/dependencies&gt;</span></code></pre>
</div>
</div>
<div class="paragraph">
<p>同样，对于maven项目的构建，可以引入公有的插件：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="xml"><span></span>    <span class="tok-nt">&lt;build&gt;</span>
        <span class="tok-nt">&lt;plugins&gt;</span>
            <span class="tok-nt">&lt;plugin&gt;</span>
                <span class="tok-nt">&lt;groupId&gt;</span>org.apache.maven.plugins<span class="tok-nt">&lt;/groupId&gt;</span>
                <span class="tok-nt">&lt;artifactId&gt;</span>maven-compiler-plugin<span class="tok-nt">&lt;/artifactId&gt;</span>
                <span class="tok-nt">&lt;version&gt;</span>3.6.1<span class="tok-nt">&lt;/version&gt;</span>
                <span class="tok-nt">&lt;configuration&gt;</span>
                    <span class="tok-nt">&lt;source&gt;</span>1.8<span class="tok-nt">&lt;/source&gt;</span>
                    <span class="tok-nt">&lt;target&gt;</span>1.8<span class="tok-nt">&lt;/target&gt;</span>
                <span class="tok-nt">&lt;/configuration&gt;</span>
            <span class="tok-nt">&lt;/plugin&gt;</span>
        <span class="tok-nt">&lt;/plugins&gt;</span>
        <span class="tok-nt">&lt;pluginManagement&gt;</span>
            <span class="tok-nt">&lt;plugins&gt;</span>
                <span class="tok-nt">&lt;plugin&gt;</span>
                    <span class="tok-nt">&lt;groupId&gt;</span>org.apache.maven.plugins<span class="tok-nt">&lt;/groupId&gt;</span>
                    <span class="tok-nt">&lt;artifactId&gt;</span>maven-assembly-plugin<span class="tok-nt">&lt;/artifactId&gt;</span>
                    <span class="tok-nt">&lt;version&gt;</span>3.0.0<span class="tok-nt">&lt;/version&gt;</span>
                    <span class="tok-nt">&lt;executions&gt;</span>
                        <span class="tok-nt">&lt;execution&gt;</span>
                            <span class="tok-nt">&lt;id&gt;</span>make-assembly<span class="tok-nt">&lt;/id&gt;</span>
                            <span class="tok-nt">&lt;phase&gt;</span>package<span class="tok-nt">&lt;/phase&gt;</span>
                            <span class="tok-nt">&lt;goals&gt;</span>
                                <span class="tok-nt">&lt;goal&gt;</span>single<span class="tok-nt">&lt;/goal&gt;</span>
                            <span class="tok-nt">&lt;/goals&gt;</span>
                        <span class="tok-nt">&lt;/execution&gt;</span>
                    <span class="tok-nt">&lt;/executions&gt;</span>
                <span class="tok-nt">&lt;/plugin&gt;</span>
                <span class="tok-nt">&lt;plugin&gt;</span>
                    <span class="tok-nt">&lt;groupId&gt;</span>net.alchim31.maven<span class="tok-nt">&lt;/groupId&gt;</span>
                    <span class="tok-nt">&lt;artifactId&gt;</span>scala-maven-plugin<span class="tok-nt">&lt;/artifactId&gt;</span>
                    <span class="tok-nt">&lt;version&gt;</span>3.2.2<span class="tok-nt">&lt;/version&gt;</span>
                    <span class="tok-nt">&lt;executions&gt;</span>
                        <span class="tok-nt">&lt;execution&gt;</span>
                            <span class="tok-nt">&lt;goals&gt;</span>
                                <span class="tok-nt">&lt;goal&gt;</span>compile<span class="tok-nt">&lt;/goal&gt;</span>
                                <span class="tok-nt">&lt;goal&gt;</span>testCompile<span class="tok-nt">&lt;/goal&gt;</span>
                            <span class="tok-nt">&lt;/goals&gt;</span>
                        <span class="tok-nt">&lt;/execution&gt;</span>
                    <span class="tok-nt">&lt;/executions&gt;</span>
                <span class="tok-nt">&lt;/plugin&gt;</span>
            <span class="tok-nt">&lt;/plugins&gt;</span>
        <span class="tok-nt">&lt;/pluginManagement&gt;</span>
    <span class="tok-nt">&lt;/build&gt;</span></code></pre>
</div>
</div>
<div class="paragraph">
<p>然后，在recommender模块中，我们可以为所有的推荐模块声明spark相关依赖（这里的dependencyManagement表示仅声明相关信息，子项目如果依赖需要自行引入）：</p>
</div>
<div class="paragraph">
<p><code>ECommerceRecommendSystem/recommender/pom.xml</code></p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="xml"><span></span>    <span class="tok-nt">&lt;dependencyManagement&gt;</span>
        <span class="tok-nt">&lt;dependencies&gt;</span>

            <span class="tok-c">&lt;!-- 引入Spark相关的Jar包 --&gt;</span>
            <span class="tok-nt">&lt;dependency&gt;</span>
                <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
                <span class="tok-nt">&lt;artifactId&gt;</span>spark-core_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
                <span class="tok-nt">&lt;version&gt;</span>2.1.1<span class="tok-nt">&lt;/version&gt;</span>
                <span class="tok-c">&lt;!-- provider如果存在，那么运行时该Jar包不存在，也不会打包到最终的发布版本中，只是编译器有效 --&gt;</span>
                <span class="tok-c">&lt;!--&lt;scope&gt;provided&lt;/scope&gt;--&gt;</span>
            <span class="tok-nt">&lt;/dependency&gt;</span>
            <span class="tok-nt">&lt;dependency&gt;</span>
                <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
                <span class="tok-nt">&lt;artifactId&gt;</span>spark-sql_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
                <span class="tok-nt">&lt;version&gt;</span>2.1.1<span class="tok-nt">&lt;/version&gt;</span>
                <span class="tok-c">&lt;!-- provider如果存在，那么运行时该Jar包不存在，也不会打包到最终的发布版本中，只是编译器有效 --&gt;</span>
                <span class="tok-c">&lt;!--&lt;scope&gt;provided&lt;/scope&gt;--&gt;</span>
            <span class="tok-nt">&lt;/dependency&gt;</span>
            <span class="tok-nt">&lt;dependency&gt;</span>
                <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
                <span class="tok-nt">&lt;artifactId&gt;</span>spark-streaming_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
                <span class="tok-nt">&lt;version&gt;</span>2.1.1<span class="tok-nt">&lt;/version&gt;</span>
                <span class="tok-c">&lt;!-- provider如果存在，那么运行时该Jar包不存在，也不会打包到最终的发布版本中，只是编译器有效 --&gt;</span>
                <span class="tok-c">&lt;!--&lt;scope&gt;provided&lt;/scope&gt;--&gt;</span>
            <span class="tok-nt">&lt;/dependency&gt;</span>
            <span class="tok-nt">&lt;dependency&gt;</span>
                <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
                <span class="tok-nt">&lt;artifactId&gt;</span>spark-mllib_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
                <span class="tok-nt">&lt;version&gt;</span>2.1.1<span class="tok-nt">&lt;/version&gt;</span>
                <span class="tok-c">&lt;!-- provider如果存在，那么运行时该Jar包不存在，也不会打包到最终的发布版本中，只是编译器有效 --&gt;</span>
                <span class="tok-c">&lt;!--&lt;scope&gt;provided&lt;/scope&gt;--&gt;</span>
            <span class="tok-nt">&lt;/dependency&gt;</span>
            <span class="tok-nt">&lt;dependency&gt;</span>
                <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
                <span class="tok-nt">&lt;artifactId&gt;</span>spark-graphx_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
                <span class="tok-nt">&lt;version&gt;</span>2.1.1<span class="tok-nt">&lt;/version&gt;</span>
                <span class="tok-c">&lt;!-- provider如果存在，那么运行时该Jar包不存在，也不会打包到最终的发布版本中，只是编译器有效 --&gt;</span>
                <span class="tok-c">&lt;!--&lt;scope&gt;provided&lt;/scope&gt;--&gt;</span>
            <span class="tok-nt">&lt;/dependency&gt;</span>

            <span class="tok-c">&lt;!-- 加入MongoDB的驱动 --&gt;</span>
            <span class="tok-c">&lt;!-- 用于代码方式连接MongoDB --&gt;</span>
            <span class="tok-nt">&lt;dependency&gt;</span>
                <span class="tok-nt">&lt;groupId&gt;</span>org.mongodb<span class="tok-nt">&lt;/groupId&gt;</span>
                <span class="tok-nt">&lt;artifactId&gt;</span>casbah-core_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
                <span class="tok-nt">&lt;version&gt;</span>${casbah.version}<span class="tok-nt">&lt;/version&gt;</span>
            <span class="tok-nt">&lt;/dependency&gt;</span>
            <span class="tok-c">&lt;!-- 用于Spark和MongoDB的对接 --&gt;</span>
            <span class="tok-nt">&lt;dependency&gt;</span>
                <span class="tok-nt">&lt;groupId&gt;</span>org.mongodb.spark<span class="tok-nt">&lt;/groupId&gt;</span>
                <span class="tok-nt">&lt;artifactId&gt;</span>mongo-spark-connector_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
                <span class="tok-nt">&lt;version&gt;</span>${mongodb-spark.version}<span class="tok-nt">&lt;/version&gt;</span>
            <span class="tok-nt">&lt;/dependency&gt;</span>

            <span class="tok-c">&lt;!-- 引入Scala --&gt;</span>
            <span class="tok-nt">&lt;dependency&gt;</span>
                <span class="tok-nt">&lt;groupId&gt;</span>org.scala-lang<span class="tok-nt">&lt;/groupId&gt;</span>
                <span class="tok-nt">&lt;artifactId&gt;</span>scala-library<span class="tok-nt">&lt;/artifactId&gt;</span>
                <span class="tok-nt">&lt;version&gt;</span>${scala.version}<span class="tok-nt">&lt;/version&gt;</span>
            <span class="tok-nt">&lt;/dependency&gt;</span>

            <span class="tok-nt">&lt;dependency&gt;</span>
                <span class="tok-nt">&lt;groupId&gt;</span>org.scalanlp<span class="tok-nt">&lt;/groupId&gt;</span>
                <span class="tok-nt">&lt;artifactId&gt;</span>jblas<span class="tok-nt">&lt;/artifactId&gt;</span>
                <span class="tok-nt">&lt;version&gt;</span>${jblas.version}<span class="tok-nt">&lt;/version&gt;</span>
            <span class="tok-nt">&lt;/dependency&gt;</span>

        <span class="tok-nt">&lt;/dependencies&gt;</span>
    <span class="tok-nt">&lt;/dependencyManagement&gt;</span></code></pre>
</div>
</div>
<div class="paragraph">
<p>由于各推荐模块都是scala代码，还应该引入scala-maven-plugin插件，用于scala程序的编译。因为插件已经在父项目中声明，所以这里不需要再声明版本和具体配置：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="xml"><span></span>    <span class="tok-nt">&lt;build&gt;</span>
        <span class="tok-nt">&lt;plugins&gt;</span>
            <span class="tok-c">&lt;!-- 如果父项目有声明plugin，那么子项目在引入的时候，不用声明版本和父项目已经声明的配置 --&gt;</span>
            <span class="tok-nt">&lt;plugin&gt;</span>
                <span class="tok-nt">&lt;groupId&gt;</span>net.alchim31.maven<span class="tok-nt">&lt;/groupId&gt;</span>
                <span class="tok-nt">&lt;artifactId&gt;</span>scala-maven-plugin<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;/plugin&gt;</span>
        <span class="tok-nt">&lt;/plugins&gt;</span>
    <span class="tok-nt">&lt;/build&gt;</span></code></pre>
</div>
</div>
<div class="paragraph">
<p>对于具体的DataLoader子项目，需要spark相关组件，还需要mongodb的相关依赖，我们在pom.xml文件中引入所有依赖（在父项目中已声明的不需要再加详细信息）：</p>
</div>
<div class="paragraph">
<p><code>ECommerceRecommendSystem/recommender/DataLoader/pom.xml</code></p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="xml"><span></span>    <span class="tok-nt">&lt;dependencies&gt;</span>
        <span class="tok-c">&lt;!-- Spark的依赖引入 --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>spark-core_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>spark-sql_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-c">&lt;!-- 引入Scala --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.scala-lang<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>scala-library<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${scala.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>


        <span class="tok-c">&lt;!-- 加入MongoDB的驱动 --&gt;</span>
        <span class="tok-c">&lt;!-- 用于代码方式连接MongoDB --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.mongodb<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>casbah-core_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${casbah.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-c">&lt;!-- 用于Spark和MongoDB的对接 --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.mongodb.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>mongo-spark-connector_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${mongodb-spark.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>

        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.logging.log4j<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>log4j-core<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>2.9.1<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.logging.log4j<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>log4j-api<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>2.9.1<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
    <span class="tok-nt">&lt;/dependencies&gt;</span></code></pre>
</div>
</div>
</div>
</div>
<div class="sect2">
<h3 id="_数据加载准备">5.6. 数据加载准备</h3>
<div class="paragraph">
<p>在src/main/目录下，可以看到已有的默认源文件目录是java，我们可以将其改名为scala。将数据文件products.csv，ratings.csv复制到资源文件目录src/main/resources下，我们将从这里读取数据并加载到mongodb中。</p>
</div>
<div class="sect3">
<h4 id="_products数据集">5.6.1. Products数据集</h4>
<div class="paragraph">
<p>数据格式：productId, name, categoryIds, amazonId, imageUrl, categories, tags</p>
</div>
<div class="paragraph">
<p>例如：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code><span></span>3982^Fuhlen 富勒 M8眩光舞者时尚节能无线鼠标(草绿)(眩光.悦动.时尚炫舞鼠标 12个月免换电池 高精度光学寻迹引擎 超细微接收器10米传输距离)^1057,439,736^B009EJN4T2^https://images-cn-4.ssl-images-amazon.com/images/I/31QPvUDNavL._SY300_QL70_.jpg^外设产品|鼠标|电脑/办公^富勒|鼠标|电子产品|好用|外观漂亮</code></pre>
</div>
</div>
<div class="paragraph">
<p>Product数据集有7个字段，每个字段之间通过“^”符号进行分割。我们用到的字段为：</p>
</div>
<div class="paragraph">
<p>字段名|字段类型|字段描述|字段备注
-----|-------|------|-------
productId|Int|商品ID
name|String|商品名称
categories|String|商品分类
imageUrl|String|商品图片
tags|String|UGC标签</p>
</div>
</div>
<div class="sect3">
<h4 id="_ratings数据集">5.6.2. Ratings数据集</h4>
<div class="paragraph">
<p>数据格式：
userId,ProductId,rating,timestamp</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code><span></span>e.g.
1,31,2.5,1260759144</code></pre>
</div>
</div>
<div class="paragraph">
<p>Rating数据集有4个字段, 每个字段之间通过“,”分割。</p>
</div>
<div class="paragraph">
<p>字段名|字段类型|字段描述|字段备注
-----|-------|------|-------
userId|Int|用户ID
productId|Int|商品ID
score|Double|评分
timestamp|Long|评分的时间</p>
</div>
</div>
<div class="sect3">
<h4 id="_日志管理配置文件">5.6.3. 日志管理配置文件</h4>
<div class="paragraph">
<p>log4j对日志的管理，需要通过配置文件来生效。在src/main/resources下新建配置文件log4j.properties，写入以下内容：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="properties"><span></span><span class="tok-na">log4j.rootLogger</span><span class="tok-o">=</span><span class="tok-s">info, stdout</span>
<span class="tok-na">log4j.appender.stdout</span><span class="tok-o">=</span><span class="tok-s">org.apache.log4j.ConsoleAppender</span>
<span class="tok-na">log4j.appender.stdout.layout</span><span class="tok-o">=</span><span class="tok-s">org.apache.log4j.PatternLayout</span>
<span class="tok-na">log4j.appender.stdout.layout.ConversionPattern</span><span class="tok-o">=</span><span class="tok-s">%d{yyyy-MM-dd HH:mm:ss,SSS}  %5p --- [%50t]  %-80c(line:%5L)  :  %m%n</span>

<span class="tok-na">log4j.appender.R</span><span class="tok-o">=</span><span class="tok-s">org.apache.log4j.RollingFileAppender</span>
<span class="tok-na">log4j.appender.R.File</span><span class="tok-o">=</span><span class="tok-s">../log/agent.log</span>
<span class="tok-na">log4j.appender.R.MaxFileSize</span><span class="tok-o">=</span><span class="tok-s">1024KB</span>
<span class="tok-na">log4j.appender.R.MaxBackupIndex</span><span class="tok-o">=</span><span class="tok-s">1</span>

<span class="tok-na">log4j.appender.R.layout</span><span class="tok-o">=</span><span class="tok-s">org.apache.log4j.PatternLayout</span>
<span class="tok-na">log4j.appender.R.layout.ConversionPattern</span><span class="tok-o">=</span><span class="tok-s">%d{yyyy-MM-dd HH:mm:ss,SSS}  %5p --- [%50t]  %-80c(line:%6L)  :  %m%n</span></code></pre>
</div>
</div>
</div>
</div>
<div class="sect2">
<h3 id="_数据初始化到mongodb">5.7. 数据初始化到MongoDB</h3>
<div class="sect3">
<h4 id="_启动mongodb数据库略">5.7.1. 启动MongoDB数据库（略）</h4>

</div>
<div class="sect3">
<h4 id="_数据加载程序主体实现">5.7.2. 数据加载程序主体实现</h4>
<div class="paragraph">
<p>我们会为原始数据定义几个样例类，通过SparkContext的textFile方法从文件中读取数据，并转换成DataFrame，再利用Spark SQL提供的write方法进行数据的分布式插入。</p>
</div>
<div class="paragraph">
<p>在DataLoader/src/main/scala下新建package，命名为com.atguigu.dataloader，新建名为DataLoader的scala class文件。</p>
</div>
<div class="paragraph">
<p>程序主体代码如下：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="scala"><span></span><span class="tok-k">package</span> <span class="tok-nn">com.atguigu.dataloader</span>

<span class="tok-k">import</span> <span class="tok-nn">com.mongodb.casbah.commons.MongoDBObject</span>
<span class="tok-k">import</span> <span class="tok-nn">com.mongodb.casbah.</span><span class="tok-o">{</span><span class="tok-nc">MongoClient</span><span class="tok-o">,</span> <span class="tok-nc">MongoClientURI</span><span class="tok-o">}</span>
<span class="tok-k">import</span> <span class="tok-nn">org.apache.spark.SparkConf</span>
<span class="tok-k">import</span> <span class="tok-nn">org.apache.spark.sql.</span><span class="tok-o">{</span><span class="tok-nc">DataFrame</span><span class="tok-o">,</span> <span class="tok-nc">SparkSession</span><span class="tok-o">}</span>

<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">Product</span><span class="tok-o">(</span><span class="tok-n">productId</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">name</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">,</span> <span class="tok-n">categories</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">,</span> <span class="tok-n">imageUrl</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">,</span> <span class="tok-n">tags</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">)</span>

<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">Rating</span><span class="tok-o">(</span><span class="tok-n">userId</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">productId</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">score</span><span class="tok-k">:</span> <span class="tok-kt">Double</span><span class="tok-o">,</span> <span class="tok-n">timestamp</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">)</span>

<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">MongoConfig</span><span class="tok-o">(</span><span class="tok-n">uri</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">,</span> <span class="tok-n">db</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">)</span>

<span class="tok-c1">// 数据的主加载服务</span>
<span class="tok-k">object</span> <span class="tok-nc">DataLoader</span> <span class="tok-o">{</span>

  <span class="tok-c1">// products.csv和ratings.csv数据集的绝对路径</span>
  <span class="tok-k">val</span> <span class="tok-nc">PRODUCTS_DATA_PATH</span> <span class="tok-k">=</span> <span class="tok-s">&quot;/Users/yuanzuo/Desktop/ECommerceRecommender/recommender/DataLoader/src/main/resources/products.csv&quot;</span>
  <span class="tok-k">val</span> <span class="tok-nc">RATING_DATA_PATH</span> <span class="tok-k">=</span> <span class="tok-s">&quot;/Users/yuanzuo/Desktop/ECommerceRecommender/recommender/DataLoader/src/main/resources/ratings.csv&quot;</span>

  <span class="tok-k">val</span> <span class="tok-nc">MONGODB_PRODUCT_COLLECTION</span> <span class="tok-k">=</span> <span class="tok-s">&quot;Products&quot;</span>
  <span class="tok-k">val</span> <span class="tok-nc">MONGODB_RATING_COLLECTION</span> <span class="tok-k">=</span> <span class="tok-s">&quot;Rating&quot;</span>

  <span class="tok-c1">// 程序的入口</span>
  <span class="tok-k">def</span> <span class="tok-n">main</span><span class="tok-o">(</span><span class="tok-n">args</span><span class="tok-k">:</span> <span class="tok-kt">Array</span><span class="tok-o">[</span><span class="tok-kt">String</span><span class="tok-o">])</span><span class="tok-k">:</span> <span class="tok-kt">Unit</span> <span class="tok-o">=</span> <span class="tok-o">{</span>

    <span class="tok-k">val</span> <span class="tok-n">config</span> <span class="tok-k">=</span> <span class="tok-nc">Map</span><span class="tok-o">(</span>
      <span class="tok-s">&quot;spark.cores&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-s">&quot;local[*]&quot;</span><span class="tok-o">,</span>
      <span class="tok-s">&quot;mongo.uri&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-s">&quot;mongodb://localhost:27017/recommender&quot;</span><span class="tok-o">,</span>
      <span class="tok-s">&quot;mongo.db&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-s">&quot;recommender&quot;</span>
    <span class="tok-o">)</span>

    <span class="tok-k">val</span> <span class="tok-n">sparkConf</span> <span class="tok-k">=</span> <span class="tok-k">new</span> <span class="tok-nc">SparkConf</span><span class="tok-o">().</span><span class="tok-n">setAppName</span><span class="tok-o">(</span><span class="tok-s">&quot;DataLoader&quot;</span><span class="tok-o">).</span><span class="tok-n">setMaster</span><span class="tok-o">(</span><span class="tok-n">config</span><span class="tok-o">.</span><span class="tok-n">get</span><span class="tok-o">(</span><span class="tok-s">&quot;spark.cores&quot;</span><span class="tok-o">).</span><span class="tok-n">get</span><span class="tok-o">)</span>

    <span class="tok-k">val</span> <span class="tok-n">spark</span> <span class="tok-k">=</span> <span class="tok-nc">SparkSession</span><span class="tok-o">.</span><span class="tok-n">builder</span><span class="tok-o">().</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-n">sparkConf</span><span class="tok-o">).</span><span class="tok-n">getOrCreate</span><span class="tok-o">()</span>

    <span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">sparkContext</span><span class="tok-o">.</span><span class="tok-n">setLogLevel</span><span class="tok-o">(</span><span class="tok-s">&quot;ERROR&quot;</span><span class="tok-o">)</span>

    <span class="tok-k">import</span> <span class="tok-nn">spark.implicits._</span>

    <span class="tok-k">val</span> <span class="tok-n">productRDD</span> <span class="tok-k">=</span> <span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">sparkContext</span><span class="tok-o">.</span><span class="tok-n">textFile</span><span class="tok-o">(</span><span class="tok-nc">PRODUCTS_DATA_PATH</span><span class="tok-o">)</span>
    <span class="tok-k">val</span> <span class="tok-n">productDF</span> <span class="tok-k">=</span> <span class="tok-n">productRDD</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">item</span> <span class="tok-o">=&gt;{</span>
      <span class="tok-k">val</span> <span class="tok-n">attr</span> <span class="tok-k">=</span> <span class="tok-n">item</span><span class="tok-o">.</span><span class="tok-n">split</span><span class="tok-o">(</span><span class="tok-s">&quot;\\^&quot;</span><span class="tok-o">)</span>
      <span class="tok-nc">Product</span><span class="tok-o">(</span><span class="tok-n">attr</span><span class="tok-o">(</span><span class="tok-mi">0</span><span class="tok-o">).</span><span class="tok-n">toInt</span><span class="tok-o">,</span> <span class="tok-n">attr</span><span class="tok-o">(</span><span class="tok-mi">1</span><span class="tok-o">).</span><span class="tok-n">trim</span><span class="tok-o">,</span> <span class="tok-n">attr</span><span class="tok-o">(</span><span class="tok-mi">5</span><span class="tok-o">).</span><span class="tok-n">trim</span><span class="tok-o">,</span> <span class="tok-n">attr</span><span class="tok-o">(</span><span class="tok-mi">4</span><span class="tok-o">).</span><span class="tok-n">trim</span><span class="tok-o">,</span> <span class="tok-n">attr</span><span class="tok-o">(</span><span class="tok-mi">6</span><span class="tok-o">).</span><span class="tok-n">trim</span><span class="tok-o">)</span>
    <span class="tok-o">}).</span><span class="tok-n">toDF</span><span class="tok-o">()</span>

    <span class="tok-k">val</span> <span class="tok-n">ratingRDD</span> <span class="tok-k">=</span> <span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">sparkContext</span><span class="tok-o">.</span><span class="tok-n">textFile</span><span class="tok-o">(</span><span class="tok-nc">RATING_DATA_PATH</span><span class="tok-o">)</span>
    <span class="tok-k">val</span> <span class="tok-n">ratingDF</span> <span class="tok-k">=</span> <span class="tok-n">ratingRDD</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">item</span> <span class="tok-k">=&gt;</span> <span class="tok-o">{</span>
      <span class="tok-k">val</span> <span class="tok-n">attr</span> <span class="tok-k">=</span> <span class="tok-n">item</span><span class="tok-o">.</span><span class="tok-n">split</span><span class="tok-o">(</span><span class="tok-s">&quot;,&quot;</span><span class="tok-o">)</span>
      <span class="tok-nc">Rating</span><span class="tok-o">(</span><span class="tok-n">attr</span><span class="tok-o">(</span><span class="tok-mi">0</span><span class="tok-o">).</span><span class="tok-n">toInt</span><span class="tok-o">,</span><span class="tok-n">attr</span><span class="tok-o">(</span><span class="tok-mi">1</span><span class="tok-o">).</span><span class="tok-n">toInt</span><span class="tok-o">,</span><span class="tok-n">attr</span><span class="tok-o">(</span><span class="tok-mi">2</span><span class="tok-o">).</span><span class="tok-n">toDouble</span><span class="tok-o">,</span><span class="tok-n">attr</span><span class="tok-o">(</span><span class="tok-mi">3</span><span class="tok-o">).</span><span class="tok-n">toInt</span><span class="tok-o">)</span>
    <span class="tok-o">}).</span><span class="tok-n">toDF</span><span class="tok-o">()</span>

    <span class="tok-k">implicit</span> <span class="tok-k">val</span> <span class="tok-n">mongoConfig</span> <span class="tok-k">=</span> <span class="tok-nc">MongoConfig</span><span class="tok-o">(</span><span class="tok-n">config</span><span class="tok-o">.</span><span class="tok-n">get</span><span class="tok-o">(</span><span class="tok-s">&quot;mongo.uri&quot;</span><span class="tok-o">).</span><span class="tok-n">get</span><span class="tok-o">,</span><span class="tok-n">config</span><span class="tok-o">.</span><span class="tok-n">get</span><span class="tok-o">(</span><span class="tok-s">&quot;mongo.db&quot;</span><span class="tok-o">).</span><span class="tok-n">get</span><span class="tok-o">)</span>

    <span class="tok-n">storeDataInMongoDB</span><span class="tok-o">(</span><span class="tok-n">productDF</span><span class="tok-o">,</span> <span class="tok-n">ratingDF</span><span class="tok-o">)</span>

    <span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">stop</span><span class="tok-o">()</span>
  <span class="tok-o">}</span>

  <span class="tok-c1">// 将数据保存到MongoDB中的方法</span>
  <span class="tok-k">def</span> <span class="tok-n">storeDataInMongoDB</span><span class="tok-o">(</span><span class="tok-n">productDF</span><span class="tok-k">:</span> <span class="tok-kt">DataFrame</span><span class="tok-o">,</span> <span class="tok-n">ratingDF</span><span class="tok-k">:</span> <span class="tok-kt">DataFrame</span><span class="tok-o">)(</span><span class="tok-k">implicit</span> <span class="tok-n">mongoConfig</span><span class="tok-k">:</span> <span class="tok-kt">MongoConfig</span><span class="tok-o">)</span><span class="tok-k">:</span> <span class="tok-kt">Unit</span> <span class="tok-o">=</span> <span class="tok-o">{</span>

    <span class="tok-c1">//新建一个到MongoDB的连接</span>
    <span class="tok-k">val</span> <span class="tok-n">mongoClient</span> <span class="tok-k">=</span> <span class="tok-nc">MongoClient</span><span class="tok-o">(</span><span class="tok-nc">MongoClientURI</span><span class="tok-o">(</span><span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">uri</span><span class="tok-o">))</span>

    <span class="tok-c1">//如果MongoDB中有对应的数据库，那么应该删除</span>
    <span class="tok-n">mongoClient</span><span class="tok-o">(</span><span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">db</span><span class="tok-o">)(</span><span class="tok-nc">MONGODB_PRODUCT_COLLECTION</span><span class="tok-o">).</span><span class="tok-n">dropCollection</span><span class="tok-o">()</span>
    <span class="tok-n">mongoClient</span><span class="tok-o">(</span><span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">db</span><span class="tok-o">)(</span><span class="tok-nc">MONGODB_RATING_COLLECTION</span><span class="tok-o">).</span><span class="tok-n">dropCollection</span><span class="tok-o">()</span>

    <span class="tok-c1">//将当前数据写入到MongoDB</span>
    <span class="tok-n">productDF</span>
      <span class="tok-o">.</span><span class="tok-n">write</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;uri&quot;</span><span class="tok-o">,</span><span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">uri</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;collection&quot;</span><span class="tok-o">,</span> <span class="tok-nc">MONGODB_PRODUCT_COLLECTION</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">mode</span><span class="tok-o">(</span><span class="tok-s">&quot;overwrite&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">format</span><span class="tok-o">(</span><span class="tok-s">&quot;com.mongodb.spark.sql&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">save</span><span class="tok-o">()</span>

    <span class="tok-n">ratingDF</span>
      <span class="tok-o">.</span><span class="tok-n">write</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;uri&quot;</span><span class="tok-o">,</span><span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">uri</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;collection&quot;</span><span class="tok-o">,</span><span class="tok-nc">MONGODB_RATING_COLLECTION</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">mode</span><span class="tok-o">(</span><span class="tok-s">&quot;overwrite&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">format</span><span class="tok-o">(</span><span class="tok-s">&quot;com.mongodb.spark.sql&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">save</span><span class="tok-o">()</span>


    <span class="tok-c1">//对数据表建索引</span>
    <span class="tok-n">mongoClient</span><span class="tok-o">(</span><span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">db</span><span class="tok-o">)(</span><span class="tok-nc">MONGODB_PRODUCT_COLLECTION</span><span class="tok-o">).</span><span class="tok-n">createIndex</span><span class="tok-o">(</span><span class="tok-nc">MongoDBObject</span><span class="tok-o">(</span><span class="tok-s">&quot;productId&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-mi">1</span><span class="tok-o">))</span>
    <span class="tok-n">mongoClient</span><span class="tok-o">(</span><span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">db</span><span class="tok-o">)(</span><span class="tok-nc">MONGODB_RATING_COLLECTION</span><span class="tok-o">).</span><span class="tok-n">createIndex</span><span class="tok-o">(</span><span class="tok-nc">MongoDBObject</span><span class="tok-o">(</span><span class="tok-s">&quot;userId&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-mi">1</span><span class="tok-o">))</span>
    <span class="tok-n">mongoClient</span><span class="tok-o">(</span><span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">db</span><span class="tok-o">)(</span><span class="tok-nc">MONGODB_RATING_COLLECTION</span><span class="tok-o">).</span><span class="tok-n">createIndex</span><span class="tok-o">(</span><span class="tok-nc">MongoDBObject</span><span class="tok-o">(</span><span class="tok-s">&quot;productId&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-mi">1</span><span class="tok-o">))</span>

    <span class="tok-c1">//关闭MongoDB的连接</span>
    <span class="tok-n">mongoClient</span><span class="tok-o">.</span><span class="tok-n">close</span><span class="tok-o">()</span>
  <span class="tok-o">}</span>
<span class="tok-o">}</span></code></pre>
</div>
</div>
</div>
</div>
<div class="sect2">
<h3 id="_离线推荐服务建设">5.8. 离线推荐服务建设</h3>
<div class="sect3">
<h4 id="_离线推荐服务">5.8.1. 离线推荐服务</h4>
<div class="paragraph">
<p>离线推荐服务是综合用户所有的历史数据，利用设定的离线统计算法和离线推荐算法周期性的进行结果统计与保存，计算的结果在一定时间周期内是固定不变的，变更的频率取决于算法调度的频率。</p>
</div>
<div class="paragraph">
<p>离线推荐服务主要计算一些可以预先进行统计和计算的指标，为实时计算和前端业务相应提供数据支撑。</p>
</div>
<div class="paragraph">
<p>离线推荐服务主要分为统计性算法、基于物品的协同过滤、基于ALS的协同过滤推荐算法、基于内容相似度的推荐。</p>
</div>
<div class="paragraph">
<p>在recommender下新建子项目StatisticsRecommender，pom.xml文件中只需引入spark、scala和mongodb的相关依赖：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="xml"><span></span>    <span class="tok-nt">&lt;dependencies&gt;</span>
        <span class="tok-c">&lt;!-- Spark的依赖引入 --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>spark-core_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>spark-sql_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-c">&lt;!-- 引入Scala --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.scala-lang<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>scala-library<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${scala.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>

        <span class="tok-c">&lt;!-- 加入MongoDB的驱动 --&gt;</span>
        <span class="tok-c">&lt;!-- 用于代码方式连接MongoDB --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.mongodb<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>casbah-core_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${casbah.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-c">&lt;!-- 用于Spark和MongoDB的对接 --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.mongodb.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>mongo-spark-connector_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${mongodb-spark.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>

    <span class="tok-nt">&lt;/dependencies&gt;</span></code></pre>
</div>
</div>
<div class="paragraph">
<p>在resources文件夹下引入log4j.properties，然后在src/main/scala下新建scala单例对象com.atguigu.statistics.StatisticsRecommender。</p>
</div>
<div class="paragraph">
<p>同样，我们应该先建好样例类，在main()方法中定义配置、创建SparkSession并加载数据，最后关闭spark。代码如下：</p>
</div>
<div class="paragraph">
<p><code>src/main/scala/com.atguigu.statistics/StatisticsRecommender.scala</code></p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="scala"><span></span><span class="tok-k">package</span> <span class="tok-nn">com.atguigu.statistic</span>

<span class="tok-k">import</span> <span class="tok-nn">java.text.SimpleDateFormat</span>
<span class="tok-k">import</span> <span class="tok-nn">java.util.Date</span>

<span class="tok-k">import</span> <span class="tok-nn">org.apache.spark.SparkConf</span>
<span class="tok-k">import</span> <span class="tok-nn">org.apache.spark.sql.SparkSession</span>

<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">Product</span><span class="tok-o">(</span><span class="tok-n">productId</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">name</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">,</span> <span class="tok-n">categories</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">,</span> <span class="tok-n">imageUrl</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">,</span> <span class="tok-n">tags</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">)</span>

<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">Rating</span><span class="tok-o">(</span><span class="tok-n">userId</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">productId</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">score</span><span class="tok-k">:</span> <span class="tok-kt">Double</span><span class="tok-o">,</span> <span class="tok-n">timestamp</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">)</span>

<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">MongoConfig</span><span class="tok-o">(</span><span class="tok-n">uri</span><span class="tok-k">:</span><span class="tok-kt">String</span><span class="tok-o">,</span> <span class="tok-n">db</span><span class="tok-k">:</span><span class="tok-kt">String</span><span class="tok-o">)</span>

<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">Recommendation</span><span class="tok-o">(</span><span class="tok-n">rid</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">r</span><span class="tok-k">:</span> <span class="tok-kt">Double</span><span class="tok-o">)</span>

<span class="tok-k">object</span> <span class="tok-nc">StatisticRecommender</span> <span class="tok-o">{</span>

  <span class="tok-k">val</span> <span class="tok-nc">MONGODB_RATING_COLLECTION</span> <span class="tok-k">=</span> <span class="tok-s">&quot;Rating&quot;</span>
  <span class="tok-k">val</span> <span class="tok-nc">MONGODB_PRODUCT_COLLECTION</span> <span class="tok-k">=</span> <span class="tok-s">&quot;Products&quot;</span>

  <span class="tok-c1">//统计的表的名称</span>
  <span class="tok-k">val</span> <span class="tok-nc">RATE_MORE_PRODUCTS</span> <span class="tok-k">=</span> <span class="tok-s">&quot;RateMoreProducts&quot;</span>
  <span class="tok-k">val</span> <span class="tok-nc">RATE_MORE_RECENTLY_PRODUCTS</span> <span class="tok-k">=</span> <span class="tok-s">&quot;RateMoreRecentlyProducts&quot;</span>
  <span class="tok-k">val</span> <span class="tok-nc">AVERAGE_PRODUCTS</span> <span class="tok-k">=</span> <span class="tok-s">&quot;AverageProducts&quot;</span>

  <span class="tok-c1">// 入口方法</span>
  <span class="tok-k">def</span> <span class="tok-n">main</span><span class="tok-o">(</span><span class="tok-n">args</span><span class="tok-k">:</span> <span class="tok-kt">Array</span><span class="tok-o">[</span><span class="tok-kt">String</span><span class="tok-o">])</span><span class="tok-k">:</span> <span class="tok-kt">Unit</span> <span class="tok-o">=</span> <span class="tok-o">{</span>

    <span class="tok-k">val</span> <span class="tok-n">config</span> <span class="tok-k">=</span> <span class="tok-nc">Map</span><span class="tok-o">(</span>
      <span class="tok-s">&quot;spark.cores&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-s">&quot;local[*]&quot;</span><span class="tok-o">,</span>
      <span class="tok-s">&quot;mongo.uri&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-s">&quot;mongodb://localhost:27017/recommender&quot;</span><span class="tok-o">,</span>
      <span class="tok-s">&quot;mongo.db&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-s">&quot;recommender&quot;</span>
    <span class="tok-o">)</span>

    <span class="tok-c1">// 创建SparkConf配置</span>
    <span class="tok-k">val</span> <span class="tok-n">sparkConf</span> <span class="tok-k">=</span> <span class="tok-k">new</span> <span class="tok-nc">SparkConf</span><span class="tok-o">().</span><span class="tok-n">setAppName</span><span class="tok-o">(</span><span class="tok-s">&quot;StatisticRecommender&quot;</span><span class="tok-o">).</span><span class="tok-n">setMaster</span><span class="tok-o">(</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-s">&quot;spark.cores&quot;</span><span class="tok-o">))</span>

    <span class="tok-c1">// 创建SparkSession</span>
    <span class="tok-k">val</span> <span class="tok-n">spark</span> <span class="tok-k">=</span> <span class="tok-nc">SparkSession</span><span class="tok-o">.</span><span class="tok-n">builder</span><span class="tok-o">().</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-n">sparkConf</span><span class="tok-o">).</span><span class="tok-n">getOrCreate</span><span class="tok-o">()</span>

    <span class="tok-c1">// 调高日志等级</span>
    <span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">sparkContext</span><span class="tok-o">.</span><span class="tok-n">setLogLevel</span><span class="tok-o">(</span><span class="tok-s">&quot;ERROR&quot;</span><span class="tok-o">)</span>

    <span class="tok-k">val</span> <span class="tok-n">mongoConfig</span> <span class="tok-k">=</span> <span class="tok-nc">MongoConfig</span><span class="tok-o">(</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-s">&quot;mongo.uri&quot;</span><span class="tok-o">),</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-s">&quot;mongo.db&quot;</span><span class="tok-o">))</span>

    <span class="tok-c1">//加入隐式转换</span>
    <span class="tok-k">import</span> <span class="tok-nn">spark.implicits._</span>

    <span class="tok-c1">//数据加载进来</span>
    <span class="tok-k">val</span> <span class="tok-n">ratingDF</span> <span class="tok-k">=</span> <span class="tok-n">spark</span>
      <span class="tok-o">.</span><span class="tok-n">read</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;uri&quot;</span><span class="tok-o">,</span> <span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">uri</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;collection&quot;</span><span class="tok-o">,</span> <span class="tok-nc">MONGODB_RATING_COLLECTION</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">format</span><span class="tok-o">(</span><span class="tok-s">&quot;com.mongodb.spark.sql&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">load</span><span class="tok-o">()</span>
      <span class="tok-o">.</span><span class="tok-n">as</span><span class="tok-o">[</span><span class="tok-kt">Rating</span><span class="tok-o">]</span>
      <span class="tok-o">.</span><span class="tok-n">toDF</span><span class="tok-o">()</span>

    <span class="tok-k">val</span> <span class="tok-n">productDF</span> <span class="tok-k">=</span> <span class="tok-n">spark</span>
      <span class="tok-o">.</span><span class="tok-n">read</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;uri&quot;</span><span class="tok-o">,</span> <span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">uri</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;collection&quot;</span><span class="tok-o">,</span> <span class="tok-nc">MONGODB_PRODUCT_COLLECTION</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">format</span><span class="tok-o">(</span><span class="tok-s">&quot;com.mongodb.spark.sql&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">load</span><span class="tok-o">()</span>
      <span class="tok-o">.</span><span class="tok-n">as</span><span class="tok-o">[</span><span class="tok-kt">Product</span><span class="tok-o">]</span>
      <span class="tok-o">.</span><span class="tok-n">toDF</span><span class="tok-o">()</span>

    <span class="tok-n">ratingDF</span><span class="tok-o">.</span><span class="tok-n">createOrReplaceTempView</span><span class="tok-o">(</span><span class="tok-s">&quot;ratings&quot;</span><span class="tok-o">)</span>

    <span class="tok-k">val</span> <span class="tok-n">rateMoreProductsDF</span> <span class="tok-k">=</span> <span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">sql</span><span class="tok-o">(</span><span class="tok-s">&quot;select productId, count(productId) as count from ratings group by productId&quot;</span><span class="tok-o">)</span>

    <span class="tok-n">rateMoreProductsDF</span>
      <span class="tok-o">.</span><span class="tok-n">write</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;uri&quot;</span><span class="tok-o">,</span> <span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">uri</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;collection&quot;</span><span class="tok-o">,</span> <span class="tok-nc">RATE_MORE_PRODUCTS</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">mode</span><span class="tok-o">(</span><span class="tok-s">&quot;overwrite&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">format</span><span class="tok-o">(</span><span class="tok-s">&quot;com.mongodb.spark.sql&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">save</span><span class="tok-o">()</span>

    <span class="tok-k">val</span> <span class="tok-n">simpleDateFormat</span> <span class="tok-k">=</span> <span class="tok-k">new</span> <span class="tok-nc">SimpleDateFormat</span><span class="tok-o">(</span><span class="tok-s">&quot;yyyyMM&quot;</span><span class="tok-o">)</span>

    <span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">udf</span><span class="tok-o">.</span><span class="tok-n">register</span><span class="tok-o">(</span><span class="tok-s">&quot;changeDate&quot;</span><span class="tok-o">,</span> <span class="tok-o">(</span><span class="tok-n">x</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">)</span> <span class="tok-k">=&gt;</span> <span class="tok-n">simpleDateFormat</span><span class="tok-o">.</span><span class="tok-n">format</span><span class="tok-o">(</span><span class="tok-k">new</span> <span class="tok-nc">Date</span><span class="tok-o">(</span><span class="tok-n">x</span> <span class="tok-o">*</span> <span class="tok-mi">1000L</span><span class="tok-o">)).</span><span class="tok-n">toInt</span><span class="tok-o">)</span>

    <span class="tok-k">val</span> <span class="tok-n">ratingOfYearMonth</span> <span class="tok-k">=</span> <span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">sql</span><span class="tok-o">(</span><span class="tok-s">&quot;select productId, score, changeDate(timestamp) as yearmonth from ratings&quot;</span><span class="tok-o">)</span>

    <span class="tok-n">ratingOfYearMonth</span><span class="tok-o">.</span><span class="tok-n">createOrReplaceTempView</span><span class="tok-o">(</span><span class="tok-s">&quot;ratingOfMonth&quot;</span><span class="tok-o">)</span>

    <span class="tok-k">val</span> <span class="tok-n">rateMoreRecentlyProducts</span> <span class="tok-k">=</span> <span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">sql</span><span class="tok-o">(</span><span class="tok-s">&quot;select productId, count(productId) as count, yearmonth from ratingOfMonth group by yearmonth, productId&quot;</span><span class="tok-o">)</span>

    <span class="tok-n">rateMoreRecentlyProducts</span>
      <span class="tok-o">.</span><span class="tok-n">write</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;uri&quot;</span><span class="tok-o">,</span><span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">uri</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;collection&quot;</span><span class="tok-o">,</span><span class="tok-nc">RATE_MORE_RECENTLY_PRODUCTS</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">mode</span><span class="tok-o">(</span><span class="tok-s">&quot;overwrite&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">format</span><span class="tok-o">(</span><span class="tok-s">&quot;com.mongodb.spark.sql&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">save</span><span class="tok-o">()</span>

    <span class="tok-k">val</span> <span class="tok-n">averageProductsDF</span> <span class="tok-k">=</span> <span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">sql</span><span class="tok-o">(</span><span class="tok-s">&quot;select productId, avg(score) as avg from ratings group by productId order by avg desc&quot;</span><span class="tok-o">)</span>

    <span class="tok-n">averageProductsDF</span><span class="tok-o">.</span><span class="tok-n">show</span><span class="tok-o">()</span>

    <span class="tok-n">averageProductsDF</span>
      <span class="tok-o">.</span><span class="tok-n">write</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;uri&quot;</span><span class="tok-o">,</span><span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">uri</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;collection&quot;</span><span class="tok-o">,</span><span class="tok-nc">AVERAGE_PRODUCTS</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">mode</span><span class="tok-o">(</span><span class="tok-s">&quot;overwrite&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">format</span><span class="tok-o">(</span><span class="tok-s">&quot;com.mongodb.spark.sql&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">save</span><span class="tok-o">()</span>

    <span class="tok-c1">//关闭Spark</span>
    <span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">stop</span><span class="tok-o">()</span>
  <span class="tok-o">}</span>
<span class="tok-o">}</span></code></pre>
</div>
</div>
</div>
<div class="sect3">
<h4 id="_离线统计服务">5.8.2. 离线统计服务</h4>
<div class="paragraph">
<p>下面我们针对以上代码分开讲解：</p>
</div>
<div class="sect4">
<h5 id="_历史热门商品统计">历史热门商品统计</h5>
<div class="paragraph">
<p>根据所有历史评分数据，计算历史评分次数最多的商品。</p>
</div>
<div class="paragraph">
<p>实现思路：</p>
</div>
<div class="paragraph">
<p>通过Spark SQL读取评分数据集，统计所有评分中评分数最多的商品，然后按照从大到小排序，将最终结果写入MongoDB的RateMoreProducts数据集中。</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="scala"><span></span><span class="tok-c1">//统计所有历史数据中每个商品的评分数</span>
<span class="tok-c1">//数据结构 -&gt;  productId,count</span>
<span class="tok-k">val</span> <span class="tok-n">rateMoreProductsDF</span> <span class="tok-k">=</span> <span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">sql</span><span class="tok-o">(</span><span class="tok-s">&quot;select productId, count(productId) as count from ratings group by productId&quot;</span><span class="tok-o">)</span>

<span class="tok-n">rateMoreProductsDF</span>
    <span class="tok-o">.</span><span class="tok-n">write</span>
    <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;uri&quot;</span><span class="tok-o">,</span> <span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">uri</span><span class="tok-o">)</span>
    <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;collection&quot;</span><span class="tok-o">,</span> <span class="tok-nc">RATE_MORE_PRODUCTS</span><span class="tok-o">)</span>
    <span class="tok-o">.</span><span class="tok-n">mode</span><span class="tok-o">(</span><span class="tok-s">&quot;overwrite&quot;</span><span class="tok-o">)</span>
    <span class="tok-o">.</span><span class="tok-n">format</span><span class="tok-o">(</span><span class="tok-s">&quot;com.mongodb.spark.sql&quot;</span><span class="tok-o">)</span>
    <span class="tok-o">.</span><span class="tok-n">save</span><span class="tok-o">()</span></code></pre>
</div>
</div>
</div>
<div class="sect4">
<h5 id="_最近热门商品统计">最近热门商品统计</h5>
<div class="paragraph">
<p>根据评分，按月为单位计算最近时间的月份里面评分数最多的商品集合。</p>
</div>
<div class="paragraph">
<p>实现思路：</p>
</div>
<div class="paragraph">
<p>通过Spark SQL读取评分数据集，通过UDF函数将评分的数据时间修改为月，然后统计每月商品的评分数。统计完成之后将数据写入到MongoDB的RateMoreRecentlyProducts数据集中。</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="scala"><span></span><span class="tok-c1">//统计以月为单位拟每个电商的评分数</span>
<span class="tok-c1">//数据结构 -&gt; productId,count,time</span>

<span class="tok-c1">//创建一个日期格式化工具</span>
<span class="tok-k">val</span> <span class="tok-n">simpleDateFormat</span> <span class="tok-k">=</span> <span class="tok-k">new</span> <span class="tok-nc">SimpleDateFormat</span><span class="tok-o">(</span><span class="tok-s">&quot;yyyyMM&quot;</span><span class="tok-o">)</span>

<span class="tok-c1">//注册一个UDF函数，用于将timestamp装换成年月格式   1260759144000  =&gt; 201605</span>
<span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">udf</span><span class="tok-o">.</span><span class="tok-n">register</span><span class="tok-o">(</span><span class="tok-s">&quot;changeDate&quot;</span><span class="tok-o">,</span> <span class="tok-o">(</span><span class="tok-n">x</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">)</span> <span class="tok-k">=&gt;</span> <span class="tok-n">simpleDateFormat</span><span class="tok-o">.</span><span class="tok-n">format</span><span class="tok-o">(</span><span class="tok-k">new</span> <span class="tok-nc">Date</span><span class="tok-o">(</span><span class="tok-n">x</span> <span class="tok-o">*</span> <span class="tok-mi">1000L</span><span class="tok-o">)).</span><span class="tok-n">toInt</span><span class="tok-o">)</span>

<span class="tok-c1">// 将原来的Rating数据集中的时间转换成年月的格式</span>
<span class="tok-k">val</span> <span class="tok-n">ratingOfYearMonth</span> <span class="tok-k">=</span> <span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">sql</span><span class="tok-o">(</span><span class="tok-s">&quot;select productId, score, changeDate(timestamp) as yearmonth from ratings&quot;</span><span class="tok-o">)</span>

<span class="tok-c1">// 将新的数据集注册成为一张表</span>
<span class="tok-n">ratingOfYearMonth</span><span class="tok-o">.</span><span class="tok-n">createOrReplaceTempView</span><span class="tok-o">(</span><span class="tok-s">&quot;ratingOfMonth&quot;</span><span class="tok-o">)</span>

<span class="tok-k">val</span> <span class="tok-n">rateMoreRecentlyProducts</span> <span class="tok-k">=</span> <span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">sql</span><span class="tok-o">(</span><span class="tok-s">&quot;select productId, count(productId) as count, yearmonth from ratingOfMonth group by yearmonth, productId&quot;</span><span class="tok-o">)</span>

<span class="tok-n">rateMoreRecentlyProducts</span>
    <span class="tok-o">.</span><span class="tok-n">write</span>
    <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;uri&quot;</span><span class="tok-o">,</span><span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">uri</span><span class="tok-o">)</span>
    <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;collection&quot;</span><span class="tok-o">,</span><span class="tok-nc">RATE_MORE_RECENTLY_PRODUCTS</span><span class="tok-o">)</span>
    <span class="tok-o">.</span><span class="tok-n">mode</span><span class="tok-o">(</span><span class="tok-s">&quot;overwrite&quot;</span><span class="tok-o">)</span>
    <span class="tok-o">.</span><span class="tok-n">format</span><span class="tok-o">(</span><span class="tok-s">&quot;com.mongodb.spark.sql&quot;</span><span class="tok-o">)</span>
    <span class="tok-o">.</span><span class="tok-n">save</span><span class="tok-o">()</span></code></pre>
</div>
</div>
</div>
<div class="sect4">
<h5 id="_商品平均得分统计">商品平均得分统计</h5>
<div class="paragraph">
<p>根据历史数据中所有用户对商品的评分，周期性的计算每个商品的平均得分。</p>
</div>
<div class="paragraph">
<p>实现思路：</p>
</div>
<div class="paragraph">
<p>通过Spark SQL读取保存在MongDB中的Rating数据集，通过执行以下SQL语句实现对于商品的平均分统计：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="scala"><span></span><span class="tok-k">val</span> <span class="tok-n">averageProductsDF</span> <span class="tok-k">=</span> <span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">sql</span><span class="tok-o">(</span><span class="tok-s">&quot;select productId, avg(score) as avg from ratings group by productId order by avg desc&quot;</span><span class="tok-o">)</span>

<span class="tok-n">averageProductsDF</span><span class="tok-o">.</span><span class="tok-n">show</span><span class="tok-o">()</span>

<span class="tok-n">averageProductsDF</span>
    <span class="tok-o">.</span><span class="tok-n">write</span>
    <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;uri&quot;</span><span class="tok-o">,</span><span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">uri</span><span class="tok-o">)</span>
    <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;collection&quot;</span><span class="tok-o">,</span><span class="tok-nc">AVERAGE_PRODUCTS</span><span class="tok-o">)</span>
    <span class="tok-o">.</span><span class="tok-n">mode</span><span class="tok-o">(</span><span class="tok-s">&quot;overwrite&quot;</span><span class="tok-o">)</span>
    <span class="tok-o">.</span><span class="tok-n">format</span><span class="tok-o">(</span><span class="tok-s">&quot;com.mongodb.spark.sql&quot;</span><span class="tok-o">)</span>
    <span class="tok-o">.</span><span class="tok-n">save</span><span class="tok-o">()</span></code></pre>
</div>
</div>
</div>
</div>
<div class="sect3">
<h4 id="_基于隐语义模型的协同过滤推荐">5.8.3. 基于隐语义模型的协同过滤推荐</h4>
<div class="paragraph">
<p>项目采用ALS作为协同过滤算法，分别根据MongoDB中的用户评分表和商品数据集计算用户商品推荐矩阵以及商品相似度矩阵。</p>
</div>
<div class="sect4">
<h5 id="_用户商品推荐矩阵">用户商品推荐矩阵</h5>
<div class="paragraph">
<p>通过ALS训练出来的Model来计算所有当前用户电商的推荐矩阵，主要思路如下：</p>
</div>
<div class="olist arabic">
<ol class="arabic">
<li>
<p>UserId和ProductID做笛卡尔积，产生（userId，productId）的元组</p>
</li>
<li>
<p>通过模型预测（userId，productId）的元组。</p>
</li>
<li>
<p>将预测结果通过预测分值进行排序。</p>
</li>
<li>
<p>返回分值最大的K个电商，作为当前用户的推荐。</p>
</li>
</ol>
</div>
<div class="paragraph">
<p>最后生成的数据结构如下：将数据保存到MongoDB的UserRecs表中</p>
</div>
<div class="paragraph">
<p><strong>图略</strong></p>
</div>
<div class="paragraph">
<p>新建recommender的子项目OfflineRecommender，引入spark、scala、mongo和jblas的依赖：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="xml"><span></span>    <span class="tok-nt">&lt;dependencies&gt;</span>

        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.scalanlp<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>jblas<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${jblas.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>

        <span class="tok-c">&lt;!-- Spark的依赖引入 --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>spark-core_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>spark-sql_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>spark-mllib_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-c">&lt;!-- 引入Scala --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.scala-lang<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>scala-library<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${scala.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>

        <span class="tok-c">&lt;!-- 加入MongoDB的驱动 --&gt;</span>
        <span class="tok-c">&lt;!-- 用于代码方式连接MongoDB --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.mongodb<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>casbah-core_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${casbah.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-c">&lt;!-- 用于Spark和MongoDB的对接 --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.mongodb.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>mongo-spark-connector_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${mongodb-spark.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>

    <span class="tok-nt">&lt;/dependencies&gt;</span></code></pre>
</div>
</div>
<div class="paragraph">
<p>同样经过前期的构建样例类、声明配置、创建SparkSession等步骤，可以加载数据开始计算模型了。核心代码如下：</p>
</div>
<div class="paragraph">
<p><code>src/main/scala/com.atguigu.offline/OfflineRecommender.scala</code></p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="scala"><span></span><span class="tok-k">package</span> <span class="tok-nn">com.atguigu.offline</span>

<span class="tok-k">import</span> <span class="tok-nn">org.apache.spark.SparkConf</span>
<span class="tok-k">import</span> <span class="tok-nn">org.apache.spark.mllib.recommendation.</span><span class="tok-o">{</span><span class="tok-nc">ALS</span><span class="tok-o">,</span> <span class="tok-nc">Rating</span><span class="tok-o">}</span>
<span class="tok-k">import</span> <span class="tok-nn">org.apache.spark.sql.SparkSession</span>
<span class="tok-k">import</span> <span class="tok-nn">org.jblas.DoubleMatrix</span>

<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">Product</span><span class="tok-o">(</span><span class="tok-n">productId</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">name</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">,</span> <span class="tok-n">categories</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">,</span> <span class="tok-n">imageUrl</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">,</span> <span class="tok-n">tags</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">)</span>

<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">ProductRating</span><span class="tok-o">(</span><span class="tok-n">userId</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">productId</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">score</span><span class="tok-k">:</span> <span class="tok-kt">Double</span><span class="tok-o">,</span> <span class="tok-n">timestamp</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">)</span>

<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">MongoConfig</span><span class="tok-o">(</span><span class="tok-n">uri</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">,</span> <span class="tok-n">db</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">)</span>

<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">Recommendation</span><span class="tok-o">(</span><span class="tok-n">rid</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">r</span><span class="tok-k">:</span> <span class="tok-kt">Double</span><span class="tok-o">)</span>

<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">UserRecs</span><span class="tok-o">(</span><span class="tok-n">userId</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">recs</span><span class="tok-k">:</span> <span class="tok-kt">Seq</span><span class="tok-o">[</span><span class="tok-kt">Recommendation</span><span class="tok-o">])</span>

<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">ProductRecs</span><span class="tok-o">(</span><span class="tok-n">productId</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">recs</span><span class="tok-k">:</span> <span class="tok-kt">Seq</span><span class="tok-o">[</span><span class="tok-kt">Recommendation</span><span class="tok-o">])</span>

<span class="tok-k">object</span> <span class="tok-nc">OfflineRecommender</span> <span class="tok-o">{</span>

  <span class="tok-k">val</span> <span class="tok-nc">MONGODB_RATING_COLLECTION</span> <span class="tok-k">=</span> <span class="tok-s">&quot;Rating&quot;</span>
  <span class="tok-k">val</span> <span class="tok-nc">MONGODB_PRODUCT_COLLECTION</span> <span class="tok-k">=</span> <span class="tok-s">&quot;Products&quot;</span>

  <span class="tok-k">val</span> <span class="tok-nc">USER_MAX_RECOMMENDATION</span> <span class="tok-k">=</span> <span class="tok-mi">20</span>

  <span class="tok-k">val</span> <span class="tok-nc">USER_RECS</span> <span class="tok-k">=</span> <span class="tok-s">&quot;UserRecs&quot;</span>
  <span class="tok-k">val</span> <span class="tok-nc">PRODUCT_RECS</span> <span class="tok-k">=</span> <span class="tok-s">&quot;ProductRecs&quot;</span>

  <span class="tok-c1">//入口方法</span>
  <span class="tok-k">def</span> <span class="tok-n">main</span><span class="tok-o">(</span><span class="tok-n">args</span><span class="tok-k">:</span> <span class="tok-kt">Array</span><span class="tok-o">[</span><span class="tok-kt">String</span><span class="tok-o">])</span><span class="tok-k">:</span> <span class="tok-kt">Unit</span> <span class="tok-o">=</span> <span class="tok-o">{</span>

    <span class="tok-k">val</span> <span class="tok-n">config</span> <span class="tok-k">=</span> <span class="tok-nc">Map</span><span class="tok-o">(</span>
      <span class="tok-s">&quot;spark.cores&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-s">&quot;local[*]&quot;</span><span class="tok-o">,</span>
      <span class="tok-s">&quot;mongo.uri&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-s">&quot;mongodb://localhost:27017/recommender&quot;</span><span class="tok-o">,</span>
      <span class="tok-s">&quot;mongo.db&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-s">&quot;reommender&quot;</span>
    <span class="tok-o">)</span>

    <span class="tok-c1">//创建一个SparkConf配置</span>
    <span class="tok-k">val</span> <span class="tok-n">sparkConf</span> <span class="tok-k">=</span> <span class="tok-k">new</span> <span class="tok-nc">SparkConf</span><span class="tok-o">().</span><span class="tok-n">setAppName</span><span class="tok-o">(</span><span class="tok-s">&quot;OfflineRecommender&quot;</span><span class="tok-o">).</span><span class="tok-n">setMaster</span><span class="tok-o">(</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-s">&quot;spark.cores&quot;</span><span class="tok-o">)).</span><span class="tok-n">set</span><span class="tok-o">(</span><span class="tok-s">&quot;spark.executor.memory&quot;</span><span class="tok-o">,</span><span class="tok-s">&quot;6G&quot;</span><span class="tok-o">).</span><span class="tok-n">set</span><span class="tok-o">(</span><span class="tok-s">&quot;spark.driver.memory&quot;</span><span class="tok-o">,</span><span class="tok-s">&quot;2G&quot;</span><span class="tok-o">)</span>

    <span class="tok-c1">//基于SparkConf创建一个SparkSession</span>
    <span class="tok-k">val</span> <span class="tok-n">spark</span> <span class="tok-k">=</span> <span class="tok-nc">SparkSession</span><span class="tok-o">.</span><span class="tok-n">builder</span><span class="tok-o">().</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-n">sparkConf</span><span class="tok-o">).</span><span class="tok-n">getOrCreate</span><span class="tok-o">()</span>

    <span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">sparkContext</span><span class="tok-o">.</span><span class="tok-n">setLogLevel</span><span class="tok-o">(</span><span class="tok-s">&quot;ERROR&quot;</span><span class="tok-o">)</span>

    <span class="tok-c1">//创建一个MongoDBConfig</span>
    <span class="tok-k">val</span> <span class="tok-n">mongoConfig</span> <span class="tok-k">=</span> <span class="tok-nc">MongoConfig</span><span class="tok-o">(</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-s">&quot;mongo.uri&quot;</span><span class="tok-o">),</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-s">&quot;mongo.db&quot;</span><span class="tok-o">))</span>

    <span class="tok-k">import</span> <span class="tok-nn">spark.implicits._</span>

    <span class="tok-c1">// 读取mongoDB中的业务数据</span>
    <span class="tok-k">val</span> <span class="tok-n">ratingRDD</span> <span class="tok-k">=</span> <span class="tok-n">spark</span>
      <span class="tok-o">.</span><span class="tok-n">read</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;uri&quot;</span><span class="tok-o">,</span><span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">uri</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;collection&quot;</span><span class="tok-o">,</span><span class="tok-nc">MONGODB_RATING_COLLECTION</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">format</span><span class="tok-o">(</span><span class="tok-s">&quot;com.mongodb.spark.sql&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">load</span><span class="tok-o">()</span>
      <span class="tok-o">.</span><span class="tok-n">as</span><span class="tok-o">[</span><span class="tok-kt">ProductRating</span><span class="tok-o">]</span>
      <span class="tok-o">.</span><span class="tok-n">rdd</span>
      <span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">rating</span> <span class="tok-k">=&gt;</span> <span class="tok-o">(</span><span class="tok-n">rating</span><span class="tok-o">.</span><span class="tok-n">userId</span><span class="tok-o">,</span> <span class="tok-n">rating</span><span class="tok-o">.</span><span class="tok-n">productId</span><span class="tok-o">,</span> <span class="tok-n">rating</span><span class="tok-o">.</span><span class="tok-n">score</span><span class="tok-o">)).</span><span class="tok-n">cache</span><span class="tok-o">()</span>

    <span class="tok-c1">//用户的数据集 RDD[Int]</span>
    <span class="tok-k">val</span> <span class="tok-n">userRDD</span> <span class="tok-k">=</span> <span class="tok-n">ratingRDD</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-k">_</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">).</span><span class="tok-n">distinct</span><span class="tok-o">()</span>

    <span class="tok-c1">//电影数据集 RDD[Int]</span>
    <span class="tok-k">val</span> <span class="tok-n">productRDD</span> <span class="tok-k">=</span> <span class="tok-n">spark</span>
      <span class="tok-o">.</span><span class="tok-n">read</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;uri&quot;</span><span class="tok-o">,</span><span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">uri</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;collection&quot;</span><span class="tok-o">,</span><span class="tok-nc">MONGODB_PRODUCT_COLLECTION</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">format</span><span class="tok-o">(</span><span class="tok-s">&quot;com.mongodb.spark.sql&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">load</span><span class="tok-o">()</span>
      <span class="tok-o">.</span><span class="tok-n">as</span><span class="tok-o">[</span><span class="tok-kt">Product</span><span class="tok-o">]</span>
      <span class="tok-o">.</span><span class="tok-n">rdd</span>
      <span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-k">_</span><span class="tok-o">.</span><span class="tok-n">productId</span><span class="tok-o">).</span><span class="tok-n">cache</span><span class="tok-o">()</span>

    <span class="tok-c1">//创建训练数据集</span>

    <span class="tok-k">val</span> <span class="tok-n">trainData</span> <span class="tok-k">=</span> <span class="tok-n">ratingRDD</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">x</span> <span class="tok-k">=&gt;</span> <span class="tok-nc">Rating</span><span class="tok-o">(</span><span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">,</span><span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">,</span><span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_3</span><span class="tok-o">))</span>

    <span class="tok-c1">// r: M x N</span>
    <span class="tok-c1">// u: M x K</span>
    <span class="tok-c1">// i: K x N</span>
    <span class="tok-k">val</span> <span class="tok-o">(</span><span class="tok-n">rank</span><span class="tok-o">,</span><span class="tok-n">iterations</span><span class="tok-o">,</span><span class="tok-n">lambda</span><span class="tok-o">)</span> <span class="tok-k">=</span> <span class="tok-o">(</span><span class="tok-mi">50</span><span class="tok-o">,</span> <span class="tok-mi">5</span><span class="tok-o">,</span> <span class="tok-mf">0.01</span><span class="tok-o">)</span>
    <span class="tok-c1">//训练ALS模型</span>
    <span class="tok-k">val</span> <span class="tok-n">model</span> <span class="tok-k">=</span> <span class="tok-nc">ALS</span><span class="tok-o">.</span><span class="tok-n">train</span><span class="tok-o">(</span><span class="tok-n">trainData</span><span class="tok-o">,</span><span class="tok-n">rank</span><span class="tok-o">,</span><span class="tok-n">iterations</span><span class="tok-o">,</span><span class="tok-n">lambda</span><span class="tok-o">)</span>

    <span class="tok-c1">//计算用户推荐矩阵</span>

    <span class="tok-c1">//需要构造一个usersProducts  RDD[(Int,Int)]</span>
    <span class="tok-k">val</span> <span class="tok-n">userProducts</span> <span class="tok-k">=</span> <span class="tok-n">userRDD</span><span class="tok-o">.</span><span class="tok-n">cartesian</span><span class="tok-o">(</span><span class="tok-n">productRDD</span><span class="tok-o">)</span>

    <span class="tok-k">val</span> <span class="tok-n">preRatings</span> <span class="tok-k">=</span> <span class="tok-n">model</span><span class="tok-o">.</span><span class="tok-n">predict</span><span class="tok-o">(</span><span class="tok-n">userProducts</span><span class="tok-o">)</span>

    <span class="tok-k">val</span> <span class="tok-n">userRecs</span> <span class="tok-k">=</span> <span class="tok-n">preRatings</span>
      <span class="tok-o">.</span><span class="tok-n">filter</span><span class="tok-o">(</span><span class="tok-k">_</span><span class="tok-o">.</span><span class="tok-n">rating</span> <span class="tok-o">&gt;</span> <span class="tok-mi">0</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">rating</span> <span class="tok-k">=&gt;</span> <span class="tok-o">(</span><span class="tok-n">rating</span><span class="tok-o">.</span><span class="tok-n">user</span><span class="tok-o">,</span> <span class="tok-o">(</span><span class="tok-n">rating</span><span class="tok-o">.</span><span class="tok-n">product</span><span class="tok-o">,</span> <span class="tok-n">rating</span><span class="tok-o">.</span><span class="tok-n">rating</span><span class="tok-o">)))</span>
      <span class="tok-o">.</span><span class="tok-n">groupByKey</span><span class="tok-o">()</span>
      <span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">{</span>
        <span class="tok-k">case</span> <span class="tok-o">(</span><span class="tok-n">userId</span><span class="tok-o">,</span><span class="tok-n">recs</span><span class="tok-o">)</span> <span class="tok-k">=&gt;</span> <span class="tok-nc">UserRecs</span><span class="tok-o">(</span><span class="tok-n">userId</span><span class="tok-o">,</span> <span class="tok-n">recs</span><span class="tok-o">.</span><span class="tok-n">toList</span><span class="tok-o">.</span><span class="tok-n">sortWith</span><span class="tok-o">(</span><span class="tok-k">_</span><span class="tok-o">.</span><span class="tok-n">_2</span> <span class="tok-o">&gt;</span> <span class="tok-k">_</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">).</span><span class="tok-n">take</span><span class="tok-o">(</span><span class="tok-nc">USER_MAX_RECOMMENDATION</span><span class="tok-o">).</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">x</span> <span class="tok-k">=&gt;</span> <span class="tok-nc">Recommendation</span><span class="tok-o">(</span><span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">,</span><span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">)))</span>
      <span class="tok-o">}.</span><span class="tok-n">toDF</span><span class="tok-o">()</span>

    <span class="tok-n">userRecs</span><span class="tok-o">.</span><span class="tok-n">write</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;uri&quot;</span><span class="tok-o">,</span><span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">uri</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;collection&quot;</span><span class="tok-o">,</span><span class="tok-nc">USER_RECS</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">mode</span><span class="tok-o">(</span><span class="tok-s">&quot;overwrite&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">format</span><span class="tok-o">(</span><span class="tok-s">&quot;com.mongodb.spark.sql&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">save</span><span class="tok-o">()</span>

    <span class="tok-k">val</span> <span class="tok-n">productFeatures</span> <span class="tok-k">=</span> <span class="tok-n">model</span><span class="tok-o">.</span><span class="tok-n">productFeatures</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">{</span><span class="tok-k">case</span> <span class="tok-o">(</span><span class="tok-n">productId</span><span class="tok-o">,</span> <span class="tok-n">features</span><span class="tok-o">)</span> <span class="tok-k">=&gt;</span>
      <span class="tok-o">(</span><span class="tok-n">productId</span><span class="tok-o">,</span> <span class="tok-k">new</span> <span class="tok-nc">DoubleMatrix</span><span class="tok-o">(</span><span class="tok-n">features</span><span class="tok-o">))</span>
    <span class="tok-o">}</span>

    <span class="tok-k">val</span> <span class="tok-n">productRecs</span> <span class="tok-k">=</span> <span class="tok-n">productFeatures</span><span class="tok-o">.</span><span class="tok-n">cartesian</span><span class="tok-o">(</span><span class="tok-n">productFeatures</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">filter</span><span class="tok-o">{</span><span class="tok-k">case</span> <span class="tok-o">(</span><span class="tok-n">a</span><span class="tok-o">,</span><span class="tok-n">b</span><span class="tok-o">)</span> <span class="tok-k">=&gt;</span> <span class="tok-n">a</span><span class="tok-o">.</span><span class="tok-n">_1</span> <span class="tok-o">!=</span> <span class="tok-n">b</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">}</span>
      <span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">{</span><span class="tok-k">case</span> <span class="tok-o">(</span><span class="tok-n">a</span><span class="tok-o">,</span><span class="tok-n">b</span><span class="tok-o">)</span> <span class="tok-k">=&gt;</span>
        <span class="tok-k">val</span> <span class="tok-n">simScore</span> <span class="tok-k">=</span> <span class="tok-k">this</span><span class="tok-o">.</span><span class="tok-n">consinSim</span><span class="tok-o">(</span><span class="tok-n">a</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">,</span><span class="tok-n">b</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">)</span>
        <span class="tok-o">(</span><span class="tok-n">a</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">,(</span><span class="tok-n">b</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">,</span><span class="tok-n">simScore</span><span class="tok-o">))</span>
      <span class="tok-o">}.</span><span class="tok-n">filter</span><span class="tok-o">(</span><span class="tok-k">_</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">.</span><span class="tok-n">_2</span> <span class="tok-o">&gt;</span> <span class="tok-mf">0.6</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">groupByKey</span><span class="tok-o">()</span>
      <span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">{</span><span class="tok-k">case</span> <span class="tok-o">(</span><span class="tok-n">productId</span><span class="tok-o">,</span> <span class="tok-n">items</span><span class="tok-o">)</span> <span class="tok-k">=&gt;</span>
        <span class="tok-nc">ProductRecs</span><span class="tok-o">(</span><span class="tok-n">productId</span><span class="tok-o">,</span> <span class="tok-n">items</span><span class="tok-o">.</span><span class="tok-n">toList</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">x</span> <span class="tok-k">=&gt;</span> <span class="tok-nc">Recommendation</span><span class="tok-o">(</span><span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">,</span><span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">)))</span>
      <span class="tok-o">}.</span><span class="tok-n">toDF</span><span class="tok-o">()</span>

    <span class="tok-n">productRecs</span>
      <span class="tok-o">.</span><span class="tok-n">write</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;uri&quot;</span><span class="tok-o">,</span> <span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">uri</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;collection&quot;</span><span class="tok-o">,</span><span class="tok-nc">PRODUCT_RECS</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">mode</span><span class="tok-o">(</span><span class="tok-s">&quot;overwrite&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">format</span><span class="tok-o">(</span><span class="tok-s">&quot;com.mongodb.spark.sql&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">save</span><span class="tok-o">()</span>

    <span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">close</span><span class="tok-o">()</span>
  <span class="tok-o">}</span>

  <span class="tok-k">def</span> <span class="tok-n">consinSim</span><span class="tok-o">(</span><span class="tok-n">product1</span><span class="tok-k">:</span> <span class="tok-kt">DoubleMatrix</span><span class="tok-o">,</span> <span class="tok-n">product2</span><span class="tok-k">:</span> <span class="tok-kt">DoubleMatrix</span><span class="tok-o">)</span> <span class="tok-k">:</span> <span class="tok-kt">Double</span> <span class="tok-o">={</span>
    <span class="tok-n">product1</span><span class="tok-o">.</span><span class="tok-n">dot</span><span class="tok-o">(</span><span class="tok-n">product2</span><span class="tok-o">)</span> <span class="tok-o">/</span> <span class="tok-o">(</span> <span class="tok-n">product1</span><span class="tok-o">.</span><span class="tok-n">norm2</span><span class="tok-o">()</span>  <span class="tok-o">*</span> <span class="tok-n">product2</span><span class="tok-o">.</span><span class="tok-n">norm2</span><span class="tok-o">()</span> <span class="tok-o">)</span>
  <span class="tok-o">}</span>
<span class="tok-o">}</span></code></pre>
</div>
</div>
</div>
<div class="sect4">
<h5 id="_商品相似度矩阵">商品相似度矩阵</h5>
<div class="paragraph">
<p>通过ALS计算商品间相似度矩阵，该矩阵用于查询当前电商的相似电商并为实时推荐系统服务。</p>
</div>
<div class="paragraph">
<p>离线计算的ALS算法，算法最终会为用户、商品分别生成最终的特征矩阵，分别是表示用户特征矩阵的\(U_{M \times K}\)矩阵，每个用户由\(K\)个特征描述；表示物品特征矩阵的\(V{N \times K}\)矩阵，每个物品也由\(K\)个特征描述。</p>
</div>
<div class="paragraph">
<p>\(V_{N \times K}\)表示物品特征矩阵，每一行是一个\(K\)维向量，虽然我们并不知道每一个维度的特征意义是什么，但是\(K\)个维度的数学向量表示了该行对应电商的特征。</p>
</div>
<div class="paragraph">
<p>所以，每个商品用\(V_{N \times K}\)每一行的\((t_1, \dots, t_K)\)向量表示其特征。</p>
</div>
<div class="paragraph">
<p>于是任意两个商品</p>
</div>
<div class="paragraph">
<p>\(p\): 特征向量为\(V_p=(t_{p1}, \dots, t_{pk})\)</p>
</div>
<div class="paragraph">
<p>\(q\): 特征向量为\(V_q=(t_{q1}, \dots, t_{qk})\)</p>
</div>
<div class="paragraph">
<p>之间的相似度\(sim(p, q)\)可以使用\(V_p\)和\(V_q\)的余弦值来表示:</p>
</div>
<div class="stemblock">
<div class="content">
\[sim(p,q)=\frac{\sum_{i=0}^k(t_{pi} \times t_{qi})}{\sqrt{\sum_{i=0}^kt_{pi}^2} \times \sqrt{\sum_{i=0}^kt_{qi}^2}}\]
</div>
</div>
<div class="paragraph">
<p>数据集中任意两个商品间相似度都可以由公式计算得到，商品与商品之间的相似度在一段时间内基本是固定值。最后生成的数据保存到MongoDB的ProductRecs表中。</p>
</div>
<div class="paragraph">
<p><strong>图略</strong></p>
</div>
<div class="paragraph">
<p>核心代码如下：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="scala"><span></span>    <span class="tok-k">val</span> <span class="tok-n">productFeatures</span> <span class="tok-k">=</span> <span class="tok-n">model</span><span class="tok-o">.</span><span class="tok-n">productFeatures</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">{</span><span class="tok-k">case</span> <span class="tok-o">(</span><span class="tok-n">productId</span><span class="tok-o">,</span> <span class="tok-n">features</span><span class="tok-o">)</span> <span class="tok-k">=&gt;</span>
      <span class="tok-o">(</span><span class="tok-n">productId</span><span class="tok-o">,</span> <span class="tok-k">new</span> <span class="tok-nc">DoubleMatrix</span><span class="tok-o">(</span><span class="tok-n">features</span><span class="tok-o">))</span>
    <span class="tok-o">}</span>

    <span class="tok-k">val</span> <span class="tok-n">productRecs</span> <span class="tok-k">=</span> <span class="tok-n">productFeatures</span><span class="tok-o">.</span><span class="tok-n">cartesian</span><span class="tok-o">(</span><span class="tok-n">productFeatures</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">filter</span><span class="tok-o">{</span><span class="tok-k">case</span> <span class="tok-o">(</span><span class="tok-n">a</span><span class="tok-o">,</span><span class="tok-n">b</span><span class="tok-o">)</span> <span class="tok-k">=&gt;</span> <span class="tok-n">a</span><span class="tok-o">.</span><span class="tok-n">_1</span> <span class="tok-o">!=</span> <span class="tok-n">b</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">}</span>
      <span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">{</span><span class="tok-k">case</span> <span class="tok-o">(</span><span class="tok-n">a</span><span class="tok-o">,</span><span class="tok-n">b</span><span class="tok-o">)</span> <span class="tok-k">=&gt;</span>
        <span class="tok-k">val</span> <span class="tok-n">simScore</span> <span class="tok-k">=</span> <span class="tok-k">this</span><span class="tok-o">.</span><span class="tok-n">consinSim</span><span class="tok-o">(</span><span class="tok-n">a</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">,</span><span class="tok-n">b</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">)</span>
        <span class="tok-o">(</span><span class="tok-n">a</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">,(</span><span class="tok-n">b</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">,</span><span class="tok-n">simScore</span><span class="tok-o">))</span>
      <span class="tok-o">}.</span><span class="tok-n">filter</span><span class="tok-o">(</span><span class="tok-k">_</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">.</span><span class="tok-n">_2</span> <span class="tok-o">&gt;</span> <span class="tok-mf">0.6</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">groupByKey</span><span class="tok-o">()</span>
      <span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">{</span><span class="tok-k">case</span> <span class="tok-o">(</span><span class="tok-n">productId</span><span class="tok-o">,</span> <span class="tok-n">items</span><span class="tok-o">)</span> <span class="tok-k">=&gt;</span>
        <span class="tok-nc">ProductRecs</span><span class="tok-o">(</span><span class="tok-n">productId</span><span class="tok-o">,</span> <span class="tok-n">items</span><span class="tok-o">.</span><span class="tok-n">toList</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">x</span> <span class="tok-k">=&gt;</span> <span class="tok-nc">Recommendation</span><span class="tok-o">(</span><span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">,</span><span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">)))</span>
      <span class="tok-o">}.</span><span class="tok-n">toDF</span><span class="tok-o">()</span>

    <span class="tok-n">productRecs</span>
      <span class="tok-o">.</span><span class="tok-n">write</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;uri&quot;</span><span class="tok-o">,</span> <span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">uri</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;collection&quot;</span><span class="tok-o">,</span><span class="tok-nc">PRODUCT_RECS</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">mode</span><span class="tok-o">(</span><span class="tok-s">&quot;overwrite&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">format</span><span class="tok-o">(</span><span class="tok-s">&quot;com.mongodb.spark.sql&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">save</span><span class="tok-o">()</span></code></pre>
</div>
</div>
<div class="paragraph">
<p>其中，consinSim是求两个向量余弦相似度的函数，代码实现如下：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="scala"><span></span>  <span class="tok-k">def</span> <span class="tok-n">consinSim</span><span class="tok-o">(</span><span class="tok-n">product1</span><span class="tok-k">:</span> <span class="tok-kt">DoubleMatrix</span><span class="tok-o">,</span> <span class="tok-n">product2</span><span class="tok-k">:</span> <span class="tok-kt">DoubleMatrix</span><span class="tok-o">)</span> <span class="tok-k">:</span> <span class="tok-kt">Double</span> <span class="tok-o">={</span>
    <span class="tok-n">product1</span><span class="tok-o">.</span><span class="tok-n">dot</span><span class="tok-o">(</span><span class="tok-n">product2</span><span class="tok-o">)</span> <span class="tok-o">/</span> <span class="tok-o">(</span> <span class="tok-n">product1</span><span class="tok-o">.</span><span class="tok-n">norm2</span><span class="tok-o">()</span>  <span class="tok-o">*</span> <span class="tok-n">product2</span><span class="tok-o">.</span><span class="tok-n">norm2</span><span class="tok-o">()</span> <span class="tok-o">)</span>
  <span class="tok-o">}</span></code></pre>
</div>
</div>
</div>
<div class="sect4">
<h5 id="_模型评估和参数选取">模型评估和参数选取</h5>
<div class="paragraph">
<p>在上述模型训练的过程中，我们直接给定了隐语义模型的\(rank, iterations, lambda\)三个参数。对于我们的模型，这并不一定是最优的参数选取，所以我们需要对模型进行评估。通常的做法是计算均方根误差（RMSE），考察预测评分与实际评分之间的误差。</p>
</div>
<div class="stemblock">
<div class="content">
\[RMSE=\sqrt {\frac{1}{N}\sum_{t=1}^N(observed_t-predicted_t)^2}\]
</div>
</div>
<div class="paragraph">
<p>有了RMSE，我们可以就可以通过多次调整参数值，来选取RMSE最小的一组作为我们模型的优化选择。</p>
</div>
<div class="paragraph">
<p>在`scala/com.atguigu.offline/`下新建单例对象ALSTrainer，代码主体架构如下：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="scala"><span></span><span class="tok-k">package</span> <span class="tok-nn">com.atguigu.offline</span>

<span class="tok-k">import</span> <span class="tok-nn">breeze.numerics.sqrt</span>
<span class="tok-k">import</span> <span class="tok-nn">org.apache.spark.SparkConf</span>
<span class="tok-k">import</span> <span class="tok-nn">org.apache.spark.mllib.recommendation.</span><span class="tok-o">{</span><span class="tok-nc">ALS</span><span class="tok-o">,</span> <span class="tok-nc">MatrixFactorizationModel</span><span class="tok-o">,</span> <span class="tok-nc">Rating</span><span class="tok-o">}</span>
<span class="tok-k">import</span> <span class="tok-nn">org.apache.spark.rdd.RDD</span>
<span class="tok-k">import</span> <span class="tok-nn">org.apache.spark.sql.SparkSession</span>

<span class="tok-k">object</span> <span class="tok-nc">ALSTrainer</span> <span class="tok-o">{</span>

  <span class="tok-k">def</span> <span class="tok-n">main</span><span class="tok-o">(</span><span class="tok-n">args</span><span class="tok-k">:</span> <span class="tok-kt">Array</span><span class="tok-o">[</span><span class="tok-kt">String</span><span class="tok-o">])</span><span class="tok-k">:</span> <span class="tok-kt">Unit</span> <span class="tok-o">=</span> <span class="tok-o">{</span>

    <span class="tok-k">val</span> <span class="tok-n">config</span> <span class="tok-k">=</span> <span class="tok-nc">Map</span><span class="tok-o">(</span>
      <span class="tok-s">&quot;spark.cores&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-s">&quot;local[*]&quot;</span><span class="tok-o">,</span>
      <span class="tok-s">&quot;mongo.uri&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-s">&quot;mongodb://localhost:27017/recommender&quot;</span><span class="tok-o">,</span>
      <span class="tok-s">&quot;mongo.db&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-s">&quot;recommender&quot;</span>
    <span class="tok-o">)</span>

    <span class="tok-c1">//创建SparkConf</span>
    <span class="tok-k">val</span> <span class="tok-n">sparkConf</span> <span class="tok-k">=</span> <span class="tok-k">new</span> <span class="tok-nc">SparkConf</span><span class="tok-o">().</span><span class="tok-n">setAppName</span><span class="tok-o">(</span><span class="tok-s">&quot;ALSTrainer&quot;</span><span class="tok-o">).</span><span class="tok-n">setMaster</span><span class="tok-o">(</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-s">&quot;spark.cores&quot;</span><span class="tok-o">))</span>

    <span class="tok-c1">//创建SparkSession</span>
    <span class="tok-k">val</span> <span class="tok-n">spark</span> <span class="tok-k">=</span> <span class="tok-nc">SparkSession</span><span class="tok-o">.</span><span class="tok-n">builder</span><span class="tok-o">().</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-n">sparkConf</span><span class="tok-o">).</span><span class="tok-n">getOrCreate</span><span class="tok-o">()</span>

    <span class="tok-k">val</span> <span class="tok-n">mongoConfig</span> <span class="tok-k">=</span> <span class="tok-nc">MongoConfig</span><span class="tok-o">(</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-s">&quot;mongo.uri&quot;</span><span class="tok-o">),</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-s">&quot;mongo.db&quot;</span><span class="tok-o">))</span>

    <span class="tok-k">import</span> <span class="tok-nn">spark.implicits._</span>

    <span class="tok-c1">//加载评分数据</span>
    <span class="tok-k">val</span> <span class="tok-n">ratingRDD</span> <span class="tok-k">=</span> <span class="tok-n">spark</span>
      <span class="tok-o">.</span><span class="tok-n">read</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;uri&quot;</span><span class="tok-o">,</span><span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">uri</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;collection&quot;</span><span class="tok-o">,</span><span class="tok-nc">OfflineRecommender</span><span class="tok-o">.</span><span class="tok-nc">MONGODB_RATING_COLLECTION</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">format</span><span class="tok-o">(</span><span class="tok-s">&quot;com.mongodb.spark.sql&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">load</span><span class="tok-o">()</span>
      <span class="tok-o">.</span><span class="tok-n">as</span><span class="tok-o">[</span><span class="tok-kt">ProductRating</span><span class="tok-o">]</span>
      <span class="tok-o">.</span><span class="tok-n">rdd</span>
      <span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">rating</span> <span class="tok-k">=&gt;</span> <span class="tok-nc">Rating</span><span class="tok-o">(</span><span class="tok-n">rating</span><span class="tok-o">.</span><span class="tok-n">userId</span><span class="tok-o">,</span> <span class="tok-n">rating</span><span class="tok-o">.</span><span class="tok-n">productId</span><span class="tok-o">,</span> <span class="tok-n">rating</span><span class="tok-o">.</span><span class="tok-n">score</span><span class="tok-o">)).</span><span class="tok-n">cache</span><span class="tok-o">()</span>

    <span class="tok-c1">// 训练集的数据量是80%，测试集的数量是20%</span>
    <span class="tok-k">val</span> <span class="tok-n">splits</span> <span class="tok-k">=</span> <span class="tok-n">ratingRDD</span><span class="tok-o">.</span><span class="tok-n">randomSplit</span><span class="tok-o">(</span><span class="tok-nc">Array</span><span class="tok-o">(</span><span class="tok-mf">0.8</span><span class="tok-o">,</span> <span class="tok-mf">0.2</span><span class="tok-o">))</span>

    <span class="tok-k">val</span> <span class="tok-n">trainingRDD</span> <span class="tok-k">=</span> <span class="tok-n">splits</span><span class="tok-o">(</span><span class="tok-mi">0</span><span class="tok-o">)</span>
    <span class="tok-k">val</span> <span class="tok-n">testingRDD</span> <span class="tok-k">=</span> <span class="tok-n">splits</span><span class="tok-o">(</span><span class="tok-mi">1</span><span class="tok-o">)</span>

    <span class="tok-c1">//输出最优参数</span>
    <span class="tok-n">adjustALSParams</span><span class="tok-o">(</span><span class="tok-n">trainingRDD</span><span class="tok-o">,</span> <span class="tok-n">testingRDD</span><span class="tok-o">)</span>

    <span class="tok-c1">//关闭Spark</span>
    <span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">close</span><span class="tok-o">()</span>
  <span class="tok-o">}</span>

  <span class="tok-c1">// 输出最终的最优参数</span>
  <span class="tok-k">def</span> <span class="tok-n">adjustALSParams</span><span class="tok-o">(</span><span class="tok-n">trainData</span><span class="tok-k">:</span><span class="tok-kt">RDD</span><span class="tok-o">[</span><span class="tok-kt">Rating</span><span class="tok-o">],</span> <span class="tok-n">testData</span><span class="tok-k">:</span><span class="tok-kt">RDD</span><span class="tok-o">[</span><span class="tok-kt">Rating</span><span class="tok-o">])</span><span class="tok-k">:</span> <span class="tok-kt">Unit</span> <span class="tok-o">={</span>
    <span class="tok-k">val</span> <span class="tok-n">result</span> <span class="tok-k">=</span> <span class="tok-k">for</span><span class="tok-o">(</span><span class="tok-n">rank</span> <span class="tok-k">&lt;-</span> <span class="tok-nc">Array</span><span class="tok-o">(</span><span class="tok-mi">30</span><span class="tok-o">,</span><span class="tok-mi">40</span><span class="tok-o">,</span><span class="tok-mi">50</span><span class="tok-o">,</span><span class="tok-mi">60</span><span class="tok-o">,</span><span class="tok-mi">70</span><span class="tok-o">);</span> <span class="tok-n">lambda</span> <span class="tok-k">&lt;-</span> <span class="tok-nc">Array</span><span class="tok-o">(</span><span class="tok-mi">1</span><span class="tok-o">,</span> <span class="tok-mf">0.1</span><span class="tok-o">,</span> <span class="tok-mf">0.001</span><span class="tok-o">))</span>
      <span class="tok-k">yield</span> <span class="tok-o">{</span>
        <span class="tok-k">val</span> <span class="tok-n">model</span> <span class="tok-k">=</span> <span class="tok-nc">ALS</span><span class="tok-o">.</span><span class="tok-n">train</span><span class="tok-o">(</span><span class="tok-n">trainData</span><span class="tok-o">,</span><span class="tok-n">rank</span><span class="tok-o">,</span><span class="tok-mi">5</span><span class="tok-o">,</span><span class="tok-n">lambda</span><span class="tok-o">)</span>
        <span class="tok-k">val</span> <span class="tok-n">rmse</span> <span class="tok-k">=</span> <span class="tok-n">getRmse</span><span class="tok-o">(</span><span class="tok-n">model</span><span class="tok-o">,</span> <span class="tok-n">testData</span><span class="tok-o">)</span>
        <span class="tok-o">(</span><span class="tok-n">rank</span><span class="tok-o">,</span><span class="tok-n">lambda</span><span class="tok-o">,</span><span class="tok-n">rmse</span><span class="tok-o">)</span>
      <span class="tok-o">}</span>
    <span class="tok-n">println</span><span class="tok-o">(</span><span class="tok-n">result</span><span class="tok-o">.</span><span class="tok-n">sortBy</span><span class="tok-o">(</span><span class="tok-k">_</span><span class="tok-o">.</span><span class="tok-n">_3</span><span class="tok-o">).</span><span class="tok-n">head</span><span class="tok-o">)</span>
  <span class="tok-o">}</span>

  <span class="tok-k">def</span> <span class="tok-n">getRmse</span><span class="tok-o">(</span><span class="tok-n">model</span><span class="tok-k">:</span> <span class="tok-kt">MatrixFactorizationModel</span><span class="tok-o">,</span> <span class="tok-n">testData</span><span class="tok-k">:</span> <span class="tok-kt">RDD</span><span class="tok-o">[</span><span class="tok-kt">Rating</span><span class="tok-o">])</span><span class="tok-k">:</span><span class="tok-kt">Double</span><span class="tok-o">={</span>
    <span class="tok-c1">//需要构造一个usersProducts  RDD[(Int,Int)]</span>
    <span class="tok-k">val</span> <span class="tok-n">userProducts</span> <span class="tok-k">=</span> <span class="tok-n">testData</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">item</span> <span class="tok-k">=&gt;</span> <span class="tok-o">(</span><span class="tok-n">item</span><span class="tok-o">.</span><span class="tok-n">user</span><span class="tok-o">,</span><span class="tok-n">item</span><span class="tok-o">.</span><span class="tok-n">product</span><span class="tok-o">))</span>
    <span class="tok-k">val</span> <span class="tok-n">predictRating</span> <span class="tok-k">=</span> <span class="tok-n">model</span><span class="tok-o">.</span><span class="tok-n">predict</span><span class="tok-o">(</span><span class="tok-n">userProducts</span><span class="tok-o">)</span>

    <span class="tok-k">val</span> <span class="tok-n">real</span> <span class="tok-k">=</span> <span class="tok-n">testData</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">item</span> <span class="tok-k">=&gt;</span> <span class="tok-o">((</span><span class="tok-n">item</span><span class="tok-o">.</span><span class="tok-n">user</span><span class="tok-o">,</span><span class="tok-n">item</span><span class="tok-o">.</span><span class="tok-n">product</span><span class="tok-o">),</span><span class="tok-n">item</span><span class="tok-o">.</span><span class="tok-n">rating</span><span class="tok-o">))</span>
    <span class="tok-k">val</span> <span class="tok-n">predict</span> <span class="tok-k">=</span> <span class="tok-n">predictRating</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">item</span> <span class="tok-k">=&gt;</span> <span class="tok-o">((</span><span class="tok-n">item</span><span class="tok-o">.</span><span class="tok-n">user</span><span class="tok-o">,</span><span class="tok-n">item</span><span class="tok-o">.</span><span class="tok-n">product</span><span class="tok-o">),</span><span class="tok-n">item</span><span class="tok-o">.</span><span class="tok-n">rating</span><span class="tok-o">))</span>

    <span class="tok-n">sqrt</span><span class="tok-o">(</span>
      <span class="tok-n">real</span><span class="tok-o">.</span><span class="tok-n">join</span><span class="tok-o">(</span><span class="tok-n">predict</span><span class="tok-o">).</span><span class="tok-n">map</span><span class="tok-o">{</span><span class="tok-k">case</span> <span class="tok-o">((</span><span class="tok-n">uid</span><span class="tok-o">,</span><span class="tok-n">mid</span><span class="tok-o">),(</span><span class="tok-n">real</span><span class="tok-o">,</span><span class="tok-n">pre</span><span class="tok-o">))</span><span class="tok-k">=&gt;</span>
        <span class="tok-c1">// 真实值和预测值之间的两个差值</span>
        <span class="tok-k">val</span> <span class="tok-n">err</span> <span class="tok-k">=</span> <span class="tok-n">real</span> <span class="tok-o">-</span> <span class="tok-n">pre</span>
        <span class="tok-n">err</span> <span class="tok-o">*</span> <span class="tok-n">err</span>
      <span class="tok-o">}.</span><span class="tok-n">mean</span><span class="tok-o">()</span>
    <span class="tok-o">)</span>
  <span class="tok-o">}</span>
<span class="tok-o">}</span></code></pre>
</div>
</div>
<div class="paragraph">
<p>其中adjustALSParams方法是模型评估的核心，输入一组训练数据和测试数据，输出计算得到最小RMSE的那组参数。代码实现如下：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="scala"><span></span>  <span class="tok-c1">// 输出最终的最优参数</span>
  <span class="tok-k">def</span> <span class="tok-n">adjustALSParams</span><span class="tok-o">(</span><span class="tok-n">trainData</span><span class="tok-k">:</span><span class="tok-kt">RDD</span><span class="tok-o">[</span><span class="tok-kt">Rating</span><span class="tok-o">],</span> <span class="tok-n">testData</span><span class="tok-k">:</span><span class="tok-kt">RDD</span><span class="tok-o">[</span><span class="tok-kt">Rating</span><span class="tok-o">])</span><span class="tok-k">:</span> <span class="tok-kt">Unit</span> <span class="tok-o">=</span> <span class="tok-o">{</span>
    <span class="tok-c1">// 这里指定迭代次数为5，rank和lambda在几个值中选取调整</span>
    <span class="tok-k">val</span> <span class="tok-n">result</span> <span class="tok-k">=</span> <span class="tok-k">for</span><span class="tok-o">(</span><span class="tok-n">rank</span> <span class="tok-k">&lt;-</span> <span class="tok-nc">Array</span><span class="tok-o">(</span><span class="tok-mi">30</span><span class="tok-o">,</span> <span class="tok-mi">40</span><span class="tok-o">,</span> <span class="tok-mi">50</span><span class="tok-o">,</span> <span class="tok-mi">60</span><span class="tok-o">,</span> <span class="tok-mi">70</span><span class="tok-o">);</span> <span class="tok-n">lambda</span> <span class="tok-k">&lt;-</span> <span class="tok-nc">Array</span><span class="tok-o">(</span><span class="tok-mi">1</span><span class="tok-o">,</span> <span class="tok-mf">0.1</span><span class="tok-o">,</span> <span class="tok-mf">0.001</span><span class="tok-o">))</span>
      <span class="tok-k">yield</span> <span class="tok-o">{</span>
        <span class="tok-k">val</span> <span class="tok-n">model</span> <span class="tok-k">=</span> <span class="tok-nc">ALS</span><span class="tok-o">.</span><span class="tok-n">train</span><span class="tok-o">(</span><span class="tok-n">trainData</span><span class="tok-o">,</span> <span class="tok-n">rank</span><span class="tok-o">,</span> <span class="tok-mi">5</span><span class="tok-o">,</span> <span class="tok-n">lambda</span><span class="tok-o">)</span>
        <span class="tok-k">val</span> <span class="tok-n">rmse</span> <span class="tok-k">=</span> <span class="tok-n">getRmse</span><span class="tok-o">(</span><span class="tok-n">model</span><span class="tok-o">,</span> <span class="tok-n">testData</span><span class="tok-o">)</span>
        <span class="tok-o">(</span><span class="tok-n">rank</span><span class="tok-o">,</span><span class="tok-n">lambda</span><span class="tok-o">,</span><span class="tok-n">rmse</span><span class="tok-o">)</span>
      <span class="tok-o">}</span>
    <span class="tok-n">println</span><span class="tok-o">(</span><span class="tok-n">result</span><span class="tok-o">.</span><span class="tok-n">sortBy</span><span class="tok-o">(</span><span class="tok-k">_</span><span class="tok-o">.</span><span class="tok-n">_3</span><span class="tok-o">).</span><span class="tok-n">head</span><span class="tok-o">)</span>
  <span class="tok-o">}</span></code></pre>
</div>
</div>
<div class="paragraph">
<p>计算RMSE的函数getRMSE代码实现如下：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="scala"><span></span>  <span class="tok-k">def</span> <span class="tok-n">getRmse</span><span class="tok-o">(</span><span class="tok-n">model</span><span class="tok-k">:</span> <span class="tok-kt">MatrixFactorizationModel</span><span class="tok-o">,</span> <span class="tok-n">testData</span><span class="tok-k">:</span> <span class="tok-kt">RDD</span><span class="tok-o">[</span><span class="tok-kt">Rating</span><span class="tok-o">])</span><span class="tok-k">:</span> <span class="tok-kt">Double</span> <span class="tok-o">=</span> <span class="tok-o">{</span>
    <span class="tok-c1">//需要构造一个usersProducts  RDD[(Int,Int)]</span>
    <span class="tok-k">val</span> <span class="tok-n">userProducts</span> <span class="tok-k">=</span> <span class="tok-n">testData</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">item</span> <span class="tok-k">=&gt;</span> <span class="tok-o">(</span><span class="tok-n">item</span><span class="tok-o">.</span><span class="tok-n">user</span><span class="tok-o">,</span><span class="tok-n">item</span><span class="tok-o">.</span><span class="tok-n">product</span><span class="tok-o">))</span>
    <span class="tok-k">val</span> <span class="tok-n">predictRating</span> <span class="tok-k">=</span> <span class="tok-n">model</span><span class="tok-o">.</span><span class="tok-n">predict</span><span class="tok-o">(</span><span class="tok-n">userProducts</span><span class="tok-o">)</span>

    <span class="tok-k">val</span> <span class="tok-n">real</span> <span class="tok-k">=</span> <span class="tok-n">testData</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">item</span> <span class="tok-k">=&gt;</span> <span class="tok-o">((</span><span class="tok-n">item</span><span class="tok-o">.</span><span class="tok-n">user</span><span class="tok-o">,</span><span class="tok-n">item</span><span class="tok-o">.</span><span class="tok-n">product</span><span class="tok-o">),</span><span class="tok-n">item</span><span class="tok-o">.</span><span class="tok-n">rating</span><span class="tok-o">))</span>
    <span class="tok-k">val</span> <span class="tok-n">predict</span> <span class="tok-k">=</span> <span class="tok-n">predictRating</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">item</span> <span class="tok-k">=&gt;</span> <span class="tok-o">((</span><span class="tok-n">item</span><span class="tok-o">.</span><span class="tok-n">user</span><span class="tok-o">,</span><span class="tok-n">item</span><span class="tok-o">.</span><span class="tok-n">product</span><span class="tok-o">),</span><span class="tok-n">item</span><span class="tok-o">.</span><span class="tok-n">rating</span><span class="tok-o">))</span>

    <span class="tok-n">sqrt</span><span class="tok-o">(</span>
      <span class="tok-n">real</span><span class="tok-o">.</span><span class="tok-n">join</span><span class="tok-o">(</span><span class="tok-n">predict</span><span class="tok-o">).</span><span class="tok-n">map</span><span class="tok-o">{</span><span class="tok-k">case</span> <span class="tok-o">((</span><span class="tok-n">uid</span><span class="tok-o">,</span><span class="tok-n">mid</span><span class="tok-o">),(</span><span class="tok-n">real</span><span class="tok-o">,</span><span class="tok-n">pre</span><span class="tok-o">))</span><span class="tok-k">=&gt;</span>
        <span class="tok-c1">// 真实值和预测值之间的两个差值</span>
        <span class="tok-k">val</span> <span class="tok-n">err</span> <span class="tok-k">=</span> <span class="tok-n">real</span> <span class="tok-o">-</span> <span class="tok-n">pre</span>
        <span class="tok-n">err</span> <span class="tok-o">*</span> <span class="tok-n">err</span>
      <span class="tok-o">}.</span><span class="tok-n">mean</span><span class="tok-o">()</span>
    <span class="tok-o">)</span>
  <span class="tok-o">}</span></code></pre>
</div>
</div>
<div class="paragraph">
<p>运行代码，我们就可以得到目前数据的最优模型的超参数。</p>
</div>
</div>
</div>
</div>
<div class="sect2">
<h3 id="_实时推荐服务建设">5.9. 实时推荐服务建设</h3>
<div class="sect3">
<h4 id="_实时推荐服务">5.9.1. 实时推荐服务</h4>
<div class="paragraph">
<p>实时计算与离线计算应用于推荐系统上最大的不同在于实时计算推荐结果应该反映最近一段时间用户近期的偏好，而离线计算推荐结果则是根据用户从第一次评分起的所有评分记录来计算用户总体的偏好。</p>
</div>
<div class="paragraph">
<p>用户对物品的偏好随着时间的推移总是会改变的。比如一个用户\(u\)在某时刻对商品\(p\)给予了极高的评分，那么在近期一段时候，\(u\)极有可能很喜欢与商品\(p\)类似的其他商品; 而如果用户\(u\)在某时刻对商品\(q\)给予了极低的评分，那么在近期一段时候，\(u\)极有可能不喜欢与商品\(q\)类似的其他商品。所以对于实时推荐，当用户对一个电商进行了评价后，用户会希望推荐结果基于最近这几次评分进行一定的更新，使得推荐结果匹配用户近期的偏好，满足用户近期的口味。</p>
</div>
<div class="paragraph">
<p>如果实时推荐继续采用离线推荐中的ALS算法，由于算法运行时间巨大，不具有实时得到新的推荐结果的能力；并且由于算法本身的使用的是评分表，用户本次评分后只更新了总评分表中的一项，使得算法运行后的推荐结果与用户本次评分之前的推荐结果基本没有多少差别，从而给用户一种推荐结果一直没变化的感觉，很影响用户体验。</p>
</div>
<div class="paragraph">
<p>另外，在实时推荐中由于时间性能上要满足实时或者准实时的要求，所以算法的计算量不能太大，避免复杂、过多的计算造成用户体验的下降。鉴于此，推荐精度往往不会很高。实时推荐系统更关心推荐结果的动态变化能力，只要更新推荐结果的理由合理即可，至于推荐的精度要求则可以适当放宽。</p>
</div>
<div class="paragraph">
<p>所以对于实时推荐算法，主要有两点需求：</p>
</div>
<div class="olist arabic">
<ol class="arabic">
<li>
<p>用户本次评分后、或最近几个评分后系统可以明显的更新推荐结果；</p>
</li>
<li>
<p>计算量不大，满足响应时间上的实时或者准实时要求；</p>
</li>
</ol>
</div>
</div>
<div class="sect3">
<h4 id="_实时推荐算法设计">5.9.2. 实时推荐算法设计</h4>
<div class="paragraph">
<p>当用户\(u\)对商品\(p\)进行了评分，将触发一次对u的推荐结果的更新。由于用户\(u\)对商品\(p\)评分，对于用户\(u\)来说，他与\(p\)最相似的商品们之间的推荐强度将发生变化，所以选取与商品\(p\)最相似的\(K\)个商品作为候选商品。</p>
</div>
<div class="paragraph">
<p>每个候选商品按照“推荐优先级”这一权重作为衡量这个商品被推荐给用户\(u\)的优先级。</p>
</div>
<div class="paragraph">
<p>这些商品将根据用户\(u\)最近的若干评分计算出各自对用户\(u\)的推荐优先级，然后与上次对用户\(u\)的实时推荐结果的进行基于推荐优先级的合并、替换得到更新后的推荐结果。</p>
</div>
<div class="paragraph">
<p>具体来说：</p>
</div>
<div class="paragraph">
<p>首先，获取用户\(u\)按时间顺序最近的K个评分，记为\(R_K\)；获取商品\(p\)的最相似的\(K\)个商品集合，记为\(S\)；</p>
</div>
<div class="paragraph">
<p>然后，对于每个商品\(q\)属于\(S\)，计算其推荐优先级\(E_{uq}\)，计算公式如下：</p>
</div>
<div class="stemblock">
<div class="content">
\[E_{uq}=\frac{\sum_{r \in R_K}sim(q,r)\times R_r}{sim \underline\space sum} + log_2max(incount, 1) - log_2max(recount, 1)\]
</div>
</div>
<div class="paragraph">
<p>其中：
- \(R_r\)表示用户\(u\)对商品\(r\)的评分；
- \(sim(q,r)\)表示商品\(q\)与商品\(r\)的相似度，设定最小相似度为0.6，当商品\(q\)和商品\(r\)相似度低于0.6的阈值，则视为两者不相关并忽略；
- \(sim \underline\space sum\)表示\(q\)与\(R_K\)中商品相似度大于最小阈值的个数；
- \(incount\)表示\(R_K\)中与商品\(q\)相似的、且本身评分较高\((&gt;=3)\)的商品个数；
- \(recount\)表示\(R_K\)中与商品\(q\)相似的、且本身评分较低\((&lt;3)\)的商品个数；</p>
</div>
<div class="paragraph">
<p>公式的意义如下：</p>
</div>
<div class="paragraph">
<p>首先对于每个候选商品\(q\)，从\(u\)最近的\(K\)个评分中，找出与\(q\)相似度较高\((&gt;=0.6)\)的\(u\)已评分商品们，对于这些商品们中的每个商品\(r\)，将\(r\)与\(q\)的相似度乘以用户\(u\)对\(r\)的评分，将这些乘积计算平均数，作为用户\(u\)对商品\(q\)的评分预测即</p>
</div>
<div class="stemblock">
<div class="content">
\[\frac{\sum_{r \in R_K}sim(q,r)\times R_r}{sim \underline\space sum}\]
</div>
</div>
<div class="paragraph">
<p>然后，将\(u\)最近的\(K\)个评分中与商品\(q\)相似的、且本身评分较高\((&gt;=3)\)的商品个数记为\(incount\)，计算\(log_2max(incount, 1)\)作为商品\(q\)的“增强因子”，意义在于商品\(q\)与\(u\)的最近\(K\)个评分中的\(n\)个高评分\((&gt;=3)\)商品相似，则商品\(q\)的优先级被增加\(log_2max(incount, 1)\)。如果商品\(q\)与\(u\)的最近\(K\)个评分中相似的高评分商品越多，也就是说\(n\)越大，则商品\(q\)更应该被推荐，所以推荐优先级被增强的幅度较大；如果商品\(q\)与\(u\)的最近K个评分中相似的高评分商品越少，也就是\(n\)越小，则推荐优先级被增强的幅度较小；</p>
</div>
<div class="paragraph">
<p>而后，将\(u\)最近的\(K\)个评分中与商品\(q\)相似的、且本身评分较低\((&lt;3)\)的商品个数记为\(recount\)，计算\(log_2max(recount, 1)\)作为商品\(q\)的“削弱因子”，意义在于商品q与u的最近K个评分中的n个低评分\((&lt;3)\)商品相似，则商品q的优先级被削减\(log_2max(recount, 1)\)。如果商品\(q\)与\(u\)的最近\(K\)个评分中相似的低评分商品越多，也就是说\(n\)越大，则商品\(q\)更不应该被推荐，所以推荐优先级被减弱的幅度较大；如果商品\(q\)与\(u\)的最近\(K\)个评分中相似的低评分商品越少，也就是\(n\)越小，则推荐优先级被减弱的幅度较小；</p>
</div>
<div class="paragraph">
<p>最后，将增强因子增加到上述的预测评分中，并减去削弱因子，得到最终的\(q\)商品对于\(u\)的推荐优先级。在计算完每个候选商品\(q\)的\(E_{uq}\)后，将生成一组\((商品q的ID, q的推荐优先级)\)的列表\(updatedList\)：</p>
</div>
<div class="stemblock">
<div class="content">
\[updatedList = \bigcup\limits_{q \in S}(qID, E_{uq})\]
</div>
</div>
<div class="paragraph">
<p>而在本次为用户\(u\)实时推荐之前的上一次实时推荐结果\(Rec\)也是一组$(商品m, m的推荐优先级)$的列表，其大小也为\(K\)：</p>
</div>
<div class="stemblock">
<div class="content">
\[Rec=\bigcup\limits_{m \in Rec}(mID, E_{um}); len(Rec)==K;\]
</div>
</div>
<div class="paragraph">
<p>接下来，将\(updated_S\)与本次为\(u\)实时推荐之前的上一次实时推荐结果\(Rec\)进行基于合并、替换形成新的推荐结果\(New\space Rec\)：</p>
</div>
<div class="stemblock">
<div class="content">
\[New \space Rec = topK(i\in Rec \bigcup updatedList, cmp=E_{ui})\]
</div>
</div>
<div class="paragraph">
<p>其中，\(i\)表示\(updated_S\)与\(Rec\)的商品集合中的每个商品，\(topK\)是一个函数，表示从\(Rec\)和\(updated_S\)的并集中选择出最大的\(K\)个商品，\(cmp=E_{ui}\)表示topK函数将推荐优先级\(E_{ui}\)值最大的K个商品选出来。最终，\(New \space Rec\)即为经过用户\(u\)对商品\(p\)评分后触发的实时推荐得到的最新推荐结果。</p>
</div>
<div class="paragraph">
<p>总之，实时推荐算法流程流程基本如下：</p>
</div>
<div class="olist arabic">
<ol class="arabic">
<li>
<p>用户\(u\)对商品\(p\)进行了评分，触发了实时推荐的一次计算；</p>
</li>
<li>
<p>选出商品\(p\)最相似的\(K\)个商品作为集合\(S\)；</p>
</li>
<li>
<p>获取用户\(u\)最近时间内的\(K\)条评分，包含本次评分，作为集合\(R_K\)；</p>
</li>
<li>
<p>计算商品的推荐优先级，产生\((qID, E_{uq})\)集合\(updated_S\)；</p>
</li>
</ol>
</div>
<div class="paragraph">
<p>将\(updated_S\)与上次对用户\(u\)的推荐结果\(Rec\)利用公式(4-4)进行合并，产生新的推荐结果\(New\space Rec\)；作为最终输出。</p>
</div>
<div class="paragraph">
<p>我们在recommender下新建子项目StreamingRecommender，引入spark、scala、mongo、redis和kafka的依赖:</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="xml"><span></span>    <span class="tok-nt">&lt;dependencies&gt;</span>
        <span class="tok-c">&lt;!-- Spark的依赖引入 --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>spark-core_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>spark-sql_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>spark-streaming_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-c">&lt;!-- 引入Scala --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.scala-lang<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>scala-library<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${scala.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>

        <span class="tok-c">&lt;!-- 加入MongoDB的驱动 --&gt;</span>
        <span class="tok-c">&lt;!-- 用于代码方式连接MongoDB --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.mongodb<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>casbah-core_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${casbah.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-c">&lt;!-- 用于Spark和MongoDB的对接 --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.mongodb.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>mongo-spark-connector_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${mongodb-spark.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>

        <span class="tok-c">&lt;!-- redis --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>redis.clients<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>jedis<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>2.9.0<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>

        <span class="tok-c">&lt;!-- kafka --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.kafka<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>kafka-clients<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>0.10.2.1<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>spark-streaming-kafka-0-10_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${spark.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
    <span class="tok-nt">&lt;/dependencies&gt;</span></code></pre>
</div>
</div>
<div class="paragraph">
<p>代码中首先定义样例类和一个连接助手对象（用于建立redis和mongo连接），并在StreamingRecommender中定义一些常量：</p>
</div>
<div class="paragraph">
<p><code>src/main/scala/com.atguigu.streaming/StreamingRecommender.scala</code></p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="scala"><span></span><span class="tok-k">package</span> <span class="tok-nn">com.atguigu.streaming</span>

<span class="tok-k">import</span> <span class="tok-nn">com.mongodb.casbah.commons.MongoDBObject</span>
<span class="tok-k">import</span> <span class="tok-nn">com.mongodb.casbah.</span><span class="tok-o">{</span><span class="tok-nc">MongoClient</span><span class="tok-o">,</span> <span class="tok-nc">MongoClientURI</span><span class="tok-o">}</span>
<span class="tok-k">import</span> <span class="tok-nn">org.apache.kafka.common.serialization.StringDeserializer</span>
<span class="tok-k">import</span> <span class="tok-nn">org.apache.spark.SparkConf</span>
<span class="tok-k">import</span> <span class="tok-nn">org.apache.spark.sql.SparkSession</span>
<span class="tok-k">import</span> <span class="tok-nn">org.apache.spark.streaming.kafka010.</span><span class="tok-o">{</span><span class="tok-nc">ConsumerStrategies</span><span class="tok-o">,</span> <span class="tok-nc">KafkaUtils</span><span class="tok-o">,</span> <span class="tok-nc">LocationStrategies</span><span class="tok-o">}</span>
<span class="tok-k">import</span> <span class="tok-nn">org.apache.spark.streaming.</span><span class="tok-o">{</span><span class="tok-nc">Seconds</span><span class="tok-o">,</span> <span class="tok-nc">StreamingContext</span><span class="tok-o">}</span>
<span class="tok-k">import</span> <span class="tok-nn">redis.clients.jedis.Jedis</span>

<span class="tok-k">import</span> <span class="tok-nn">scala.collection.JavaConversions._</span>

<span class="tok-k">object</span> <span class="tok-nc">ConnHelper</span> <span class="tok-k">extends</span> <span class="tok-nc">Serializable</span><span class="tok-o">{</span>
  <span class="tok-k">lazy</span> <span class="tok-k">val</span> <span class="tok-n">jedis</span> <span class="tok-k">=</span> <span class="tok-k">new</span> <span class="tok-nc">Jedis</span><span class="tok-o">(</span><span class="tok-s">&quot;localhost&quot;</span><span class="tok-o">)</span>
  <span class="tok-k">lazy</span> <span class="tok-k">val</span> <span class="tok-n">mongoClient</span> <span class="tok-k">=</span> <span class="tok-nc">MongoClient</span><span class="tok-o">(</span><span class="tok-nc">MongoClientURI</span><span class="tok-o">(</span><span class="tok-s">&quot;mongodb://localhost:27017/recommender&quot;</span><span class="tok-o">))</span>
<span class="tok-o">}</span>

<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">MongConfig</span><span class="tok-o">(</span><span class="tok-n">uri</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">,</span> <span class="tok-n">db</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">)</span>

<span class="tok-c1">//推荐</span>
<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">Recommendation</span><span class="tok-o">(</span><span class="tok-n">rid</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">r</span><span class="tok-k">:</span> <span class="tok-kt">Double</span><span class="tok-o">)</span>

<span class="tok-c1">// 用户的推荐</span>
<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">UserRecs</span><span class="tok-o">(</span><span class="tok-n">uid</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">recs</span><span class="tok-k">:</span> <span class="tok-kt">Seq</span><span class="tok-o">[</span><span class="tok-kt">Recommendation</span><span class="tok-o">])</span>

<span class="tok-c1">//商品的相似度</span>
<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">ProductRecs</span><span class="tok-o">(</span><span class="tok-n">productId</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">recs</span><span class="tok-k">:</span> <span class="tok-kt">Seq</span><span class="tok-o">[</span><span class="tok-kt">Recommendation</span><span class="tok-o">])</span>

<span class="tok-k">object</span> <span class="tok-nc">StreamingRecommender</span> <span class="tok-o">{</span>

  <span class="tok-k">val</span> <span class="tok-nc">MAX_USER_RATINGS_NUM</span> <span class="tok-k">=</span> <span class="tok-mi">20</span>
  <span class="tok-k">val</span> <span class="tok-nc">MAX_SIM_PRODUCTS_NUM</span> <span class="tok-k">=</span> <span class="tok-mi">20</span>
  <span class="tok-k">val</span> <span class="tok-nc">MONGODB_STREAM_RECS_COLLECTION</span> <span class="tok-k">=</span> <span class="tok-s">&quot;StreamRecs&quot;</span>
  <span class="tok-k">val</span> <span class="tok-nc">MONGODB_RATING_COLLECTION</span> <span class="tok-k">=</span> <span class="tok-s">&quot;Rating&quot;</span>
  <span class="tok-k">val</span> <span class="tok-nc">MONGODB_PRODUCT_RECS_COLLECTION</span> <span class="tok-k">=</span> <span class="tok-s">&quot;ProductRecs&quot;</span>

  <span class="tok-c1">//入口方法</span>
  <span class="tok-k">def</span> <span class="tok-n">main</span><span class="tok-o">(</span><span class="tok-n">args</span><span class="tok-k">:</span> <span class="tok-kt">Array</span><span class="tok-o">[</span><span class="tok-kt">String</span><span class="tok-o">])</span><span class="tok-k">:</span> <span class="tok-kt">Unit</span> <span class="tok-o">=</span> <span class="tok-o">{</span>

    <span class="tok-k">val</span> <span class="tok-n">config</span> <span class="tok-k">=</span> <span class="tok-nc">Map</span><span class="tok-o">(</span>
      <span class="tok-s">&quot;spark.cores&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-s">&quot;local[*]&quot;</span><span class="tok-o">,</span>
      <span class="tok-s">&quot;mongo.uri&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-s">&quot;mongodb://localhost:27017/recommender&quot;</span><span class="tok-o">,</span>
      <span class="tok-s">&quot;mongo.db&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-s">&quot;recommender&quot;</span><span class="tok-o">,</span>
      <span class="tok-s">&quot;kafka.topic&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-s">&quot;recommender&quot;</span>
    <span class="tok-o">)</span>
    <span class="tok-c1">//创建一个SparkConf配置</span>
    <span class="tok-k">val</span> <span class="tok-n">sparkConf</span> <span class="tok-k">=</span> <span class="tok-k">new</span> <span class="tok-nc">SparkConf</span><span class="tok-o">().</span><span class="tok-n">setAppName</span><span class="tok-o">(</span><span class="tok-s">&quot;StreamingRecommender&quot;</span><span class="tok-o">).</span><span class="tok-n">setMaster</span><span class="tok-o">(</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-s">&quot;spark.cores&quot;</span><span class="tok-o">))</span>

    <span class="tok-c1">//创建Spark的对象, 因为spark session中没有封装streaming context，所以需要new一个</span>
    <span class="tok-k">val</span> <span class="tok-n">spark</span> <span class="tok-k">=</span> <span class="tok-nc">SparkSession</span><span class="tok-o">.</span><span class="tok-n">builder</span><span class="tok-o">().</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-n">sparkConf</span><span class="tok-o">).</span><span class="tok-n">getOrCreate</span><span class="tok-o">()</span>
    <span class="tok-k">val</span> <span class="tok-n">sc</span> <span class="tok-k">=</span> <span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">sparkContext</span>
    <span class="tok-n">sc</span><span class="tok-o">.</span><span class="tok-n">setLogLevel</span><span class="tok-o">(</span><span class="tok-s">&quot;ERROR&quot;</span><span class="tok-o">)</span>
    <span class="tok-k">val</span> <span class="tok-n">ssc</span> <span class="tok-k">=</span> <span class="tok-k">new</span> <span class="tok-nc">StreamingContext</span><span class="tok-o">(</span><span class="tok-n">sc</span><span class="tok-o">,</span><span class="tok-nc">Seconds</span><span class="tok-o">(</span><span class="tok-mi">2</span><span class="tok-o">))</span>

    <span class="tok-k">implicit</span> <span class="tok-k">val</span> <span class="tok-n">mongConfig</span> <span class="tok-k">=</span> <span class="tok-nc">MongConfig</span><span class="tok-o">(</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-s">&quot;mongo.uri&quot;</span><span class="tok-o">),</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-s">&quot;mongo.db&quot;</span><span class="tok-o">))</span>
    <span class="tok-k">import</span> <span class="tok-nn">spark.implicits._</span>

    <span class="tok-c1">//******************  广播商品相似度矩阵</span>

    <span class="tok-k">val</span> <span class="tok-n">simProductsMatrix</span> <span class="tok-k">=</span> <span class="tok-n">spark</span>
      <span class="tok-o">.</span><span class="tok-n">read</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;uri&quot;</span><span class="tok-o">,</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-s">&quot;mongo.uri&quot;</span><span class="tok-o">))</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;collection&quot;</span><span class="tok-o">,</span><span class="tok-nc">MONGODB_PRODUCT_RECS_COLLECTION</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">format</span><span class="tok-o">(</span><span class="tok-s">&quot;com.mongodb.spark.sql&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">load</span><span class="tok-o">()</span>
      <span class="tok-o">.</span><span class="tok-n">as</span><span class="tok-o">[</span><span class="tok-kt">ProductRecs</span><span class="tok-o">]</span>
      <span class="tok-o">.</span><span class="tok-n">rdd</span>
      <span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">{</span><span class="tok-n">recs</span> <span class="tok-k">=&gt;</span>
        <span class="tok-o">(</span><span class="tok-n">recs</span><span class="tok-o">.</span><span class="tok-n">productId</span><span class="tok-o">,</span> <span class="tok-n">recs</span><span class="tok-o">.</span><span class="tok-n">recs</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">x</span> <span class="tok-k">=&gt;</span> <span class="tok-o">(</span><span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">rid</span><span class="tok-o">,</span> <span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">r</span><span class="tok-o">)).</span><span class="tok-n">toMap</span><span class="tok-o">)</span>
      <span class="tok-o">}.</span><span class="tok-n">collectAsMap</span><span class="tok-o">()</span>

    <span class="tok-k">val</span> <span class="tok-n">simProductsMatrixBroadCast</span> <span class="tok-k">=</span> <span class="tok-n">sc</span><span class="tok-o">.</span><span class="tok-n">broadcast</span><span class="tok-o">(</span><span class="tok-n">simProductsMatrix</span><span class="tok-o">)</span>

    <span class="tok-k">val</span> <span class="tok-n">abc</span> <span class="tok-k">=</span> <span class="tok-n">sc</span><span class="tok-o">.</span><span class="tok-n">makeRDD</span><span class="tok-o">(</span><span class="tok-mi">1</span> <span class="tok-n">to</span> <span class="tok-mi">2</span><span class="tok-o">)</span>
    <span class="tok-n">abc</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">x</span> <span class="tok-k">=&gt;</span> <span class="tok-n">simProductsMatrixBroadCast</span><span class="tok-o">.</span><span class="tok-n">value</span><span class="tok-o">.</span><span class="tok-n">get</span><span class="tok-o">(</span><span class="tok-mi">1</span><span class="tok-o">)).</span><span class="tok-n">count</span><span class="tok-o">()</span>

    <span class="tok-c1">//******************</span>


    <span class="tok-c1">//创建到Kafka的连接</span>
    <span class="tok-k">val</span> <span class="tok-n">kafkaPara</span> <span class="tok-k">=</span> <span class="tok-nc">Map</span><span class="tok-o">(</span>
      <span class="tok-s">&quot;bootstrap.servers&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-s">&quot;localhost:9092&quot;</span><span class="tok-o">,</span>
      <span class="tok-s">&quot;key.deserializer&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-n">classOf</span><span class="tok-o">[</span><span class="tok-kt">StringDeserializer</span><span class="tok-o">],</span>
      <span class="tok-s">&quot;value.deserializer&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-n">classOf</span><span class="tok-o">[</span><span class="tok-kt">StringDeserializer</span><span class="tok-o">],</span>
      <span class="tok-s">&quot;group.id&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-s">&quot;recommender&quot;</span><span class="tok-o">,</span>
      <span class="tok-s">&quot;auto.offset.reset&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-s">&quot;latest&quot;</span>    <span class="tok-c1">//每次从kafka 消费数据，都是通过zookeeper存储的数据offset，来判断需要获取消息在消息日志里的起始位置</span>
    <span class="tok-o">)</span>

    <span class="tok-k">val</span> <span class="tok-n">kafkaStream</span> <span class="tok-k">=</span> <span class="tok-nc">KafkaUtils</span><span class="tok-o">.</span><span class="tok-n">createDirectStream</span><span class="tok-o">[</span><span class="tok-kt">String</span>,<span class="tok-kt">String</span><span class="tok-o">](</span><span class="tok-n">ssc</span><span class="tok-o">,</span><span class="tok-nc">LocationStrategies</span><span class="tok-o">.</span><span class="tok-nc">PreferConsistent</span><span class="tok-o">,</span><span class="tok-nc">ConsumerStrategies</span><span class="tok-o">.</span><span class="tok-nc">Subscribe</span><span class="tok-o">[</span><span class="tok-kt">String</span>,<span class="tok-kt">String</span><span class="tok-o">](</span><span class="tok-nc">Array</span><span class="tok-o">(</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-s">&quot;kafka.topic&quot;</span><span class="tok-o">)),</span><span class="tok-n">kafkaPara</span><span class="tok-o">))</span>

    <span class="tok-k">val</span> <span class="tok-n">ratingStream</span> <span class="tok-k">=</span> <span class="tok-n">kafkaStream</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">{</span><span class="tok-k">case</span> <span class="tok-n">msg</span> <span class="tok-k">=&gt;</span>
      <span class="tok-k">var</span> <span class="tok-n">attr</span> <span class="tok-k">=</span> <span class="tok-n">msg</span><span class="tok-o">.</span><span class="tok-n">value</span><span class="tok-o">().</span><span class="tok-n">split</span><span class="tok-o">(</span><span class="tok-s">&quot;\\|&quot;</span><span class="tok-o">)</span>            <span class="tok-c1">// split方法对. | * + ^需要转义（类似正则）</span>
      <span class="tok-o">(</span><span class="tok-n">attr</span><span class="tok-o">(</span><span class="tok-mi">0</span><span class="tok-o">).</span><span class="tok-n">toInt</span><span class="tok-o">,</span> <span class="tok-n">attr</span><span class="tok-o">(</span><span class="tok-mi">1</span><span class="tok-o">).</span><span class="tok-n">toInt</span><span class="tok-o">,</span> <span class="tok-n">attr</span><span class="tok-o">(</span><span class="tok-mi">2</span><span class="tok-o">).</span><span class="tok-n">toDouble</span><span class="tok-o">,</span> <span class="tok-n">attr</span><span class="tok-o">(</span><span class="tok-mi">3</span><span class="tok-o">).</span><span class="tok-n">toInt</span><span class="tok-o">)</span>
    <span class="tok-o">}</span>

    <span class="tok-n">ratingStream</span><span class="tok-o">.</span><span class="tok-n">foreachRDD</span><span class="tok-o">{</span><span class="tok-n">rdd</span> <span class="tok-k">=&gt;</span>
      <span class="tok-n">rdd</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">{</span><span class="tok-k">case</span> <span class="tok-o">(</span><span class="tok-n">userId</span><span class="tok-o">,</span> <span class="tok-n">productId</span><span class="tok-o">,</span> <span class="tok-n">score</span><span class="tok-o">,</span> <span class="tok-n">timestamp</span><span class="tok-o">)</span> <span class="tok-k">=&gt;</span>
        <span class="tok-n">println</span><span class="tok-o">(</span><span class="tok-s">&quot;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&quot;</span><span class="tok-o">)</span>

        <span class="tok-c1">//获取当前最近的M次商品评分</span>
        <span class="tok-k">val</span> <span class="tok-n">userRecentlyRatings</span> <span class="tok-k">=</span> <span class="tok-n">getUserRecentlyRating</span><span class="tok-o">(</span><span class="tok-nc">MAX_USER_RATINGS_NUM</span><span class="tok-o">,</span> <span class="tok-n">userId</span><span class="tok-o">,</span> <span class="tok-nc">ConnHelper</span><span class="tok-o">.</span><span class="tok-n">jedis</span><span class="tok-o">)</span>

        <span class="tok-c1">//获取商品P最相似的K个商品</span>
        <span class="tok-k">val</span> <span class="tok-n">simProducts</span> <span class="tok-k">=</span> <span class="tok-n">getTopSimProducts</span><span class="tok-o">(</span><span class="tok-nc">MAX_SIM_PRODUCTS_NUM</span><span class="tok-o">,</span> <span class="tok-n">productId</span><span class="tok-o">,</span> <span class="tok-n">userId</span><span class="tok-o">,</span> <span class="tok-n">simProductsMatrixBroadCast</span><span class="tok-o">.</span><span class="tok-n">value</span><span class="tok-o">)</span>

        <span class="tok-c1">//计算待选商品的推荐优先级</span>
        <span class="tok-k">val</span> <span class="tok-n">streamRecs</span> <span class="tok-k">=</span> <span class="tok-n">computeProductScores</span><span class="tok-o">(</span><span class="tok-n">simProductsMatrixBroadCast</span><span class="tok-o">.</span><span class="tok-n">value</span><span class="tok-o">,</span> <span class="tok-n">userRecentlyRatings</span><span class="tok-o">,</span> <span class="tok-n">simProducts</span><span class="tok-o">)</span>

        <span class="tok-c1">//将数据保存到MongoDB</span>
        <span class="tok-n">saveRecsToMongoDB</span><span class="tok-o">(</span><span class="tok-n">userId</span><span class="tok-o">,</span> <span class="tok-n">streamRecs</span><span class="tok-o">)</span>

      <span class="tok-o">}.</span><span class="tok-n">count</span><span class="tok-o">()</span>
    <span class="tok-o">}</span>

    <span class="tok-c1">//启动Streaming程序</span>
    <span class="tok-n">ssc</span><span class="tok-o">.</span><span class="tok-n">start</span><span class="tok-o">()</span>
    <span class="tok-n">ssc</span><span class="tok-o">.</span><span class="tok-n">awaitTermination</span><span class="tok-o">()</span>
  <span class="tok-o">}</span>

  <span class="tok-k">def</span> <span class="tok-n">saveRecsToMongoDB</span><span class="tok-o">(</span><span class="tok-n">uid</span><span class="tok-k">:</span><span class="tok-kt">Int</span><span class="tok-o">,</span><span class="tok-n">streamRecs</span><span class="tok-k">:</span><span class="tok-kt">Array</span><span class="tok-o">[(</span><span class="tok-kt">Int</span>,<span class="tok-kt">Double</span><span class="tok-o">)])(</span><span class="tok-k">implicit</span> <span class="tok-n">mongConfig</span><span class="tok-k">:</span> <span class="tok-kt">MongConfig</span><span class="tok-o">)</span><span class="tok-k">:</span> <span class="tok-kt">Unit</span> <span class="tok-o">={</span>
    <span class="tok-c1">//到StreamRecs的连接</span>
    <span class="tok-k">val</span> <span class="tok-n">streamRecsCollection</span> <span class="tok-k">=</span> <span class="tok-nc">ConnHelper</span><span class="tok-o">.</span><span class="tok-n">mongoClient</span><span class="tok-o">(</span><span class="tok-n">mongConfig</span><span class="tok-o">.</span><span class="tok-n">db</span><span class="tok-o">)(</span><span class="tok-nc">MONGODB_STREAM_RECS_COLLECTION</span><span class="tok-o">)</span>

    <span class="tok-n">streamRecsCollection</span><span class="tok-o">.</span><span class="tok-n">findAndRemove</span><span class="tok-o">(</span><span class="tok-nc">MongoDBObject</span><span class="tok-o">(</span><span class="tok-s">&quot;uid&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-n">uid</span><span class="tok-o">))</span>
    <span class="tok-c1">//streaRecsCollection.insert(MongoDBObject(&quot;uid&quot; -&gt; uid, &quot;recs&quot; -&gt; streamRecs.map(x=&gt; x._1+&quot;:&quot;+x._2).mkString(&quot;|&quot;)))</span>
    <span class="tok-n">streamRecsCollection</span><span class="tok-o">.</span><span class="tok-n">insert</span><span class="tok-o">(</span><span class="tok-nc">MongoDBObject</span><span class="tok-o">(</span><span class="tok-s">&quot;uid&quot;</span><span class="tok-o">-&gt;</span><span class="tok-n">uid</span><span class="tok-o">,</span> <span class="tok-s">&quot;recs&quot;</span><span class="tok-o">-&gt;</span> <span class="tok-n">streamRecs</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">x</span> <span class="tok-k">=&gt;</span> <span class="tok-nc">MongoDBObject</span><span class="tok-o">(</span><span class="tok-s">&quot;mid&quot;</span><span class="tok-o">-&gt;</span><span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">,</span> <span class="tok-s">&quot;score&quot;</span><span class="tok-o">-&gt;</span><span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">))</span> <span class="tok-o">))</span>

  <span class="tok-o">}</span>

  <span class="tok-k">def</span> <span class="tok-n">computeProductScores</span><span class="tok-o">(</span><span class="tok-n">simProducts</span><span class="tok-k">:</span> <span class="tok-kt">scala.collection.Map</span><span class="tok-o">[</span><span class="tok-kt">Int</span>,<span class="tok-kt">scala.collection.immutable.Map</span><span class="tok-o">[</span><span class="tok-kt">Int</span>,<span class="tok-kt">Double</span><span class="tok-o">]],</span><span class="tok-n">userRecentlyRatings</span><span class="tok-k">:</span><span class="tok-kt">Array</span><span class="tok-o">[(</span><span class="tok-kt">Int</span>,<span class="tok-kt">Double</span><span class="tok-o">)],</span><span class="tok-n">topSimProducts</span><span class="tok-k">:</span> <span class="tok-kt">Array</span><span class="tok-o">[</span><span class="tok-kt">Int</span><span class="tok-o">])</span><span class="tok-k">:</span> <span class="tok-kt">Array</span><span class="tok-o">[(</span><span class="tok-kt">Int</span>,<span class="tok-kt">Double</span><span class="tok-o">)]</span> <span class="tok-k">=</span> <span class="tok-o">{</span>

    <span class="tok-c1">//用于保存每一个待选商品和最近评分的每一个商品的权重得分</span>
    <span class="tok-k">val</span> <span class="tok-n">score</span> <span class="tok-k">=</span> <span class="tok-n">scala</span><span class="tok-o">.</span><span class="tok-n">collection</span><span class="tok-o">.</span><span class="tok-n">mutable</span><span class="tok-o">.</span><span class="tok-nc">ArrayBuffer</span><span class="tok-o">[(</span><span class="tok-kt">Int</span>,<span class="tok-kt">Double</span><span class="tok-o">)]()</span>

    <span class="tok-c1">//用于保存每一个商品的增强因子数</span>
    <span class="tok-k">val</span> <span class="tok-n">increMap</span> <span class="tok-k">=</span> <span class="tok-n">scala</span><span class="tok-o">.</span><span class="tok-n">collection</span><span class="tok-o">.</span><span class="tok-n">mutable</span><span class="tok-o">.</span><span class="tok-nc">HashMap</span><span class="tok-o">[</span><span class="tok-kt">Int</span>,<span class="tok-kt">Int</span><span class="tok-o">]()</span>

    <span class="tok-c1">//用于保存每一个商品的减弱因子数</span>
    <span class="tok-k">val</span> <span class="tok-n">decreMap</span> <span class="tok-k">=</span> <span class="tok-n">scala</span><span class="tok-o">.</span><span class="tok-n">collection</span><span class="tok-o">.</span><span class="tok-n">mutable</span><span class="tok-o">.</span><span class="tok-nc">HashMap</span><span class="tok-o">[</span><span class="tok-kt">Int</span>,<span class="tok-kt">Int</span><span class="tok-o">]()</span>

    <span class="tok-k">for</span> <span class="tok-o">(</span><span class="tok-n">topSimProduct</span> <span class="tok-k">&lt;-</span> <span class="tok-n">topSimProducts</span><span class="tok-o">;</span> <span class="tok-n">userRecentlyRating</span> <span class="tok-k">&lt;-</span> <span class="tok-n">userRecentlyRatings</span><span class="tok-o">){</span>
      <span class="tok-k">val</span> <span class="tok-n">simScore</span> <span class="tok-k">=</span> <span class="tok-n">getProductsSimScore</span><span class="tok-o">(</span><span class="tok-n">simProducts</span><span class="tok-o">,</span> <span class="tok-n">userRecentlyRating</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">,</span> <span class="tok-n">topSimProduct</span><span class="tok-o">)</span>
      <span class="tok-k">if</span><span class="tok-o">(</span><span class="tok-n">simScore</span> <span class="tok-o">&gt;</span> <span class="tok-mf">0.6</span><span class="tok-o">){</span>
        <span class="tok-n">score</span> <span class="tok-o">+=</span> <span class="tok-o">((</span><span class="tok-n">topSimProduct</span><span class="tok-o">,</span> <span class="tok-n">simScore</span> <span class="tok-o">*</span> <span class="tok-n">userRecentlyRating</span><span class="tok-o">.</span><span class="tok-n">_2</span> <span class="tok-o">))</span>
        <span class="tok-k">if</span><span class="tok-o">(</span><span class="tok-n">userRecentlyRating</span><span class="tok-o">.</span><span class="tok-n">_2</span> <span class="tok-o">&gt;</span> <span class="tok-mi">3</span><span class="tok-o">){</span>
          <span class="tok-n">increMap</span><span class="tok-o">(</span><span class="tok-n">topSimProduct</span><span class="tok-o">)</span> <span class="tok-k">=</span> <span class="tok-n">increMap</span><span class="tok-o">.</span><span class="tok-n">getOrDefault</span><span class="tok-o">(</span><span class="tok-n">topSimProduct</span><span class="tok-o">,</span><span class="tok-mi">0</span><span class="tok-o">)</span> <span class="tok-o">+</span> <span class="tok-mi">1</span>
        <span class="tok-o">}</span><span class="tok-k">else</span><span class="tok-o">{</span>
          <span class="tok-n">decreMap</span><span class="tok-o">(</span><span class="tok-n">topSimProduct</span><span class="tok-o">)</span> <span class="tok-k">=</span> <span class="tok-n">decreMap</span><span class="tok-o">.</span><span class="tok-n">getOrDefault</span><span class="tok-o">(</span><span class="tok-n">topSimProduct</span><span class="tok-o">,</span><span class="tok-mi">0</span><span class="tok-o">)</span> <span class="tok-o">+</span> <span class="tok-mi">1</span>
        <span class="tok-o">}</span>
      <span class="tok-o">}</span>
    <span class="tok-o">}</span>

    <span class="tok-n">score</span><span class="tok-o">.</span><span class="tok-n">groupBy</span><span class="tok-o">(</span><span class="tok-k">_</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">).</span><span class="tok-n">map</span><span class="tok-o">{</span><span class="tok-k">case</span> <span class="tok-o">(</span><span class="tok-n">mid</span><span class="tok-o">,</span><span class="tok-n">sims</span><span class="tok-o">)</span> <span class="tok-k">=&gt;</span>
      <span class="tok-o">(</span><span class="tok-n">mid</span><span class="tok-o">,</span><span class="tok-n">sims</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-k">_</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">).</span><span class="tok-n">sum</span> <span class="tok-o">/</span> <span class="tok-n">sims</span><span class="tok-o">.</span><span class="tok-n">length</span> <span class="tok-o">+</span> <span class="tok-n">log</span><span class="tok-o">(</span><span class="tok-n">increMap</span><span class="tok-o">.</span><span class="tok-n">getOrDefault</span><span class="tok-o">(</span><span class="tok-n">mid</span><span class="tok-o">,</span> <span class="tok-mi">1</span><span class="tok-o">))</span> <span class="tok-o">-</span> <span class="tok-n">log</span><span class="tok-o">(</span><span class="tok-n">decreMap</span><span class="tok-o">.</span><span class="tok-n">getOrDefault</span><span class="tok-o">(</span><span class="tok-n">mid</span><span class="tok-o">,</span> <span class="tok-mi">1</span><span class="tok-o">)))</span>
    <span class="tok-o">}.</span><span class="tok-n">toArray</span>

  <span class="tok-o">}</span>

  <span class="tok-c1">//取2的对数</span>
  <span class="tok-k">def</span> <span class="tok-n">log</span><span class="tok-o">(</span><span class="tok-n">m</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">)</span><span class="tok-k">:</span> <span class="tok-kt">Double</span> <span class="tok-o">={</span>
    <span class="tok-n">math</span><span class="tok-o">.</span><span class="tok-n">log</span><span class="tok-o">(</span><span class="tok-n">m</span><span class="tok-o">)</span> <span class="tok-o">/</span> <span class="tok-n">math</span><span class="tok-o">.</span><span class="tok-n">log</span><span class="tok-o">(</span><span class="tok-mi">2</span><span class="tok-o">)</span>
  <span class="tok-o">}</span>

  <span class="tok-k">def</span> <span class="tok-n">getProductsSimScore</span><span class="tok-o">(</span><span class="tok-n">simProducts</span><span class="tok-k">:</span> <span class="tok-kt">scala.collection.Map</span><span class="tok-o">[</span><span class="tok-kt">Int</span>,<span class="tok-kt">scala.collection.immutable.Map</span><span class="tok-o">[</span><span class="tok-kt">Int</span>,<span class="tok-kt">Double</span><span class="tok-o">]],</span> <span class="tok-n">userRatingProduct</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">topSimProduct</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">)</span><span class="tok-k">:</span> <span class="tok-kt">Double</span> <span class="tok-o">={</span>
    <span class="tok-n">simProducts</span><span class="tok-o">.</span><span class="tok-n">get</span><span class="tok-o">(</span><span class="tok-n">topSimProduct</span><span class="tok-o">)</span> <span class="tok-k">match</span> <span class="tok-o">{</span>
      <span class="tok-k">case</span> <span class="tok-nc">Some</span><span class="tok-o">(</span><span class="tok-n">sim</span><span class="tok-o">)</span> <span class="tok-k">=&gt;</span> <span class="tok-n">sim</span><span class="tok-o">.</span><span class="tok-n">get</span><span class="tok-o">(</span><span class="tok-n">userRatingProduct</span><span class="tok-o">)</span> <span class="tok-k">match</span> <span class="tok-o">{</span>
        <span class="tok-k">case</span> <span class="tok-nc">Some</span><span class="tok-o">(</span><span class="tok-n">score</span><span class="tok-o">)</span> <span class="tok-k">=&gt;</span> <span class="tok-n">score</span>
        <span class="tok-k">case</span> <span class="tok-nc">None</span> <span class="tok-k">=&gt;</span> <span class="tok-mf">0.0</span>
      <span class="tok-o">}</span>
      <span class="tok-k">case</span> <span class="tok-nc">None</span> <span class="tok-k">=&gt;</span> <span class="tok-mf">0.0</span>
    <span class="tok-o">}</span>
  <span class="tok-o">}</span>

  <span class="tok-k">def</span> <span class="tok-n">getTopSimProducts</span><span class="tok-o">(</span><span class="tok-n">num</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">productId</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">userId</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">simProducts</span><span class="tok-k">:</span><span class="tok-kt">scala.collection.Map</span><span class="tok-o">[</span><span class="tok-kt">Int</span>,<span class="tok-kt">scala.collection.immutable.Map</span><span class="tok-o">[</span><span class="tok-kt">Int</span>,<span class="tok-kt">Double</span><span class="tok-o">]])(</span><span class="tok-k">implicit</span> <span class="tok-n">mongConfig</span><span class="tok-k">:</span> <span class="tok-kt">MongConfig</span><span class="tok-o">)</span><span class="tok-k">:</span> <span class="tok-kt">Array</span><span class="tok-o">[</span><span class="tok-kt">Int</span><span class="tok-o">]</span> <span class="tok-o">={</span>
    <span class="tok-c1">//从广播变量的商品相似度矩阵中获取当前商品所有的相似商品</span>
    <span class="tok-k">val</span> <span class="tok-n">allSimProducts</span> <span class="tok-k">=</span> <span class="tok-n">simProducts</span><span class="tok-o">(</span><span class="tok-n">productId</span><span class="tok-o">).</span><span class="tok-n">toArray</span>
    <span class="tok-c1">//获取用户已经观看过得商品</span>
    <span class="tok-k">val</span> <span class="tok-n">ratingExist</span> <span class="tok-k">=</span> <span class="tok-nc">ConnHelper</span><span class="tok-o">.</span><span class="tok-n">mongoClient</span><span class="tok-o">(</span><span class="tok-n">mongConfig</span><span class="tok-o">.</span><span class="tok-n">db</span><span class="tok-o">)(</span><span class="tok-nc">MONGODB_RATING_COLLECTION</span><span class="tok-o">).</span><span class="tok-n">find</span><span class="tok-o">(</span><span class="tok-nc">MongoDBObject</span><span class="tok-o">(</span><span class="tok-s">&quot;userId&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-n">userId</span><span class="tok-o">)).</span><span class="tok-n">toArray</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">{</span><span class="tok-n">item</span> <span class="tok-k">=&gt;</span>
      <span class="tok-n">item</span><span class="tok-o">.</span><span class="tok-n">get</span><span class="tok-o">(</span><span class="tok-s">&quot;productId&quot;</span><span class="tok-o">).</span><span class="tok-n">toString</span><span class="tok-o">.</span><span class="tok-n">toInt</span>
    <span class="tok-o">}</span>
    <span class="tok-c1">//过滤掉已经评分过得商品，并排序输出</span>
    <span class="tok-n">allSimProducts</span><span class="tok-o">.</span><span class="tok-n">filter</span><span class="tok-o">(</span><span class="tok-n">x</span> <span class="tok-k">=&gt;</span> <span class="tok-o">!</span><span class="tok-n">ratingExist</span><span class="tok-o">.</span><span class="tok-n">contains</span><span class="tok-o">(</span><span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">)).</span><span class="tok-n">sortWith</span><span class="tok-o">(</span><span class="tok-k">_</span><span class="tok-o">.</span><span class="tok-n">_2</span> <span class="tok-o">&gt;</span> <span class="tok-k">_</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">).</span><span class="tok-n">take</span><span class="tok-o">(</span><span class="tok-n">num</span><span class="tok-o">).</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">x</span> <span class="tok-k">=&gt;</span> <span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">)</span>
  <span class="tok-o">}</span>

  <span class="tok-k">def</span> <span class="tok-n">getUserRecentlyRating</span><span class="tok-o">(</span><span class="tok-n">num</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">userId</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">jedis</span><span class="tok-k">:</span> <span class="tok-kt">Jedis</span><span class="tok-o">)</span><span class="tok-k">:</span> <span class="tok-kt">Array</span><span class="tok-o">[(</span><span class="tok-kt">Int</span>, <span class="tok-kt">Double</span><span class="tok-o">)]</span> <span class="tok-o">={</span>
    <span class="tok-c1">//从用户的队列中取出num个评论</span>
    <span class="tok-n">jedis</span><span class="tok-o">.</span><span class="tok-n">lrange</span><span class="tok-o">(</span><span class="tok-s">&quot;userId:&quot;</span> <span class="tok-o">+</span> <span class="tok-n">userId</span><span class="tok-o">.</span><span class="tok-n">toString</span><span class="tok-o">,</span> <span class="tok-mi">0</span><span class="tok-o">,</span> <span class="tok-n">num</span><span class="tok-o">).</span><span class="tok-n">map</span><span class="tok-o">{</span><span class="tok-n">item</span> <span class="tok-k">=&gt;</span>
      <span class="tok-k">val</span> <span class="tok-n">attr</span> <span class="tok-k">=</span> <span class="tok-n">item</span><span class="tok-o">.</span><span class="tok-n">split</span><span class="tok-o">(</span><span class="tok-s">&quot;\\:&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">(</span><span class="tok-n">attr</span><span class="tok-o">(</span><span class="tok-mi">0</span><span class="tok-o">).</span><span class="tok-n">trim</span><span class="tok-o">.</span><span class="tok-n">toInt</span><span class="tok-o">,</span> <span class="tok-n">attr</span><span class="tok-o">(</span><span class="tok-mi">1</span><span class="tok-o">).</span><span class="tok-n">trim</span><span class="tok-o">.</span><span class="tok-n">toDouble</span><span class="tok-o">)</span>
    <span class="tok-o">}.</span><span class="tok-n">toArray</span>
  <span class="tok-o">}</span>
<span class="tok-o">}</span></code></pre>
</div>
</div>
</div>
<div class="sect3">
<h4 id="_实时推荐算法的讲解">5.9.3. 实时推荐算法的讲解</h4>
<div class="paragraph">
<p>实时推荐算法的前提：</p>
</div>
<div class="olist arabic">
<ol class="arabic">
<li>
<p>在Redis集群中存储了每一个用户最近对商品的K次评分。实时算法可以快速获取。</p>
</li>
<li>
<p>离线推荐算法已经将商品相似度矩阵提前计算到了MongoDB中。</p>
</li>
<li>
<p>Kafka已经获取到了用户实时的评分数据。</p>
</li>
</ol>
</div>
<div class="paragraph">
<p>算法过程如下：</p>
</div>
<div class="paragraph">
<p>实时推荐算法输入为一个评分(userId, ProductId, rate, timestamp)，而执行的核心内容包括：获取userId最近K次评分、获取productId最相似K个商品、计算候选商品的推荐优先级、更新对userId的实时推荐结果。</p>
</div>
<div class="sect4">
<h5 id="_获取用户的k次最近评分">获取用户的K次最近评分</h5>
<div class="paragraph">
<p>业务服务器在接收用户评分的时候，默认会将该评分情况以userId, productId, rating, timestamp的格式插入到Redis中该用户对应的队列当中，在实时算法中，只需要通过Redis客户端获取相对应的队列内容即可。</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="scala"><span></span><span class="tok-k">import</span> <span class="tok-nn">scala.collection.JavaConversions._</span>
<span class="tok-cm">/**</span>
<span class="tok-cm">  * 获取当前最近的M次商品评分</span>
<span class="tok-cm">  * @param num  评分的个数</span>
<span class="tok-cm">  * @param userId  谁的评分</span>
<span class="tok-cm">  * @return</span>
<span class="tok-cm">  */</span>
<span class="tok-k">def</span> <span class="tok-n">getUserRecentlyRating</span><span class="tok-o">(</span><span class="tok-n">num</span><span class="tok-k">:</span><span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">userId</span><span class="tok-k">:</span><span class="tok-kt">Int</span><span class="tok-o">,</span><span class="tok-n">jedis</span><span class="tok-k">:</span><span class="tok-kt">Jedis</span><span class="tok-o">)</span><span class="tok-k">:</span> <span class="tok-kt">Array</span><span class="tok-o">[(</span><span class="tok-kt">Int</span>,<span class="tok-kt">Double</span><span class="tok-o">)]</span> <span class="tok-o">={</span>
  <span class="tok-c1">//从用户的队列中取出num个评分</span>
  <span class="tok-n">jedis</span><span class="tok-o">.</span><span class="tok-n">lrange</span><span class="tok-o">(</span><span class="tok-s">&quot;userId:&quot;</span><span class="tok-o">+</span><span class="tok-n">userId</span><span class="tok-o">.</span><span class="tok-n">toString</span><span class="tok-o">,</span> <span class="tok-mi">0</span><span class="tok-o">,</span> <span class="tok-n">num</span><span class="tok-o">).</span><span class="tok-n">map</span><span class="tok-o">{</span><span class="tok-n">item</span> <span class="tok-k">=&gt;</span>
    <span class="tok-k">val</span> <span class="tok-n">attr</span> <span class="tok-k">=</span> <span class="tok-n">item</span><span class="tok-o">.</span><span class="tok-n">split</span><span class="tok-o">(</span><span class="tok-s">&quot;\\:&quot;</span><span class="tok-o">)</span>
    <span class="tok-o">(</span><span class="tok-n">attr</span><span class="tok-o">(</span><span class="tok-mi">0</span><span class="tok-o">).</span><span class="tok-n">trim</span><span class="tok-o">.</span><span class="tok-n">toInt</span><span class="tok-o">,</span> <span class="tok-n">attr</span><span class="tok-o">(</span><span class="tok-mi">1</span><span class="tok-o">).</span><span class="tok-n">trim</span><span class="tok-o">.</span><span class="tok-n">toDouble</span><span class="tok-o">)</span>
  <span class="tok-o">}.</span><span class="tok-n">toArray</span>
<span class="tok-o">}</span></code></pre>
</div>
</div>
</div>
<div class="sect4">
<h5 id="_获取当前商品最相似的k个商品">获取当前商品最相似的K个商品</h5>
<div class="paragraph">
<p>在离线算法中，已经预先将商品的相似度矩阵进行了计算，所以每个商品productId的最相似的K个商品很容易获取：从MongoDB中读取ProductRecs数据，从productId在simHash对应的子哈希表中获取相似度前K大的那些商品。输出是数据类型为Array[Int]的数组，表示与productId最相似的商品集合，并命名为candidateProducts以作为候选商品集合。</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="scala"><span></span><span class="tok-cm">/**</span>
<span class="tok-cm">  * 获取当前商品K个相似的商品</span>
<span class="tok-cm">  * @param num          相似商品的数量</span>
<span class="tok-cm">  * @param productId          当前商品的ID</span>
<span class="tok-cm">  * @param userId          当前的评分用户</span>
<span class="tok-cm">  * @param simProducts    商品相似度矩阵的广播变量值</span>
<span class="tok-cm">  * @param mongConfig   MongoDB的配置</span>
<span class="tok-cm">  * @return</span>
<span class="tok-cm">  */</span>
<span class="tok-k">def</span> <span class="tok-n">getTopSimProducts</span><span class="tok-o">(</span><span class="tok-n">num</span><span class="tok-k">:</span><span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">productId</span><span class="tok-k">:</span><span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">userId</span><span class="tok-k">:</span><span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">simProducts</span><span class="tok-k">:</span><span class="tok-kt">scala.collection.Map</span><span class="tok-o">[</span><span class="tok-kt">Int</span>,<span class="tok-kt">scala.collection.immutable.Map</span><span class="tok-o">[</span><span class="tok-kt">Int</span>,<span class="tok-kt">Double</span><span class="tok-o">]])(</span><span class="tok-k">implicit</span> <span class="tok-n">mongConfig</span><span class="tok-k">:</span> <span class="tok-kt">MongConfig</span><span class="tok-o">)</span><span class="tok-k">:</span> <span class="tok-kt">Array</span><span class="tok-o">[</span><span class="tok-kt">Int</span><span class="tok-o">]</span> <span class="tok-o">={</span>
  <span class="tok-c1">//从广播变量的商品相似度矩阵中获取当前商品所有的相似商品</span>
  <span class="tok-k">val</span> <span class="tok-n">allSimProducts</span> <span class="tok-k">=</span> <span class="tok-n">simProducts</span><span class="tok-o">.</span><span class="tok-n">get</span><span class="tok-o">(</span><span class="tok-n">productId</span><span class="tok-o">).</span><span class="tok-n">get</span><span class="tok-o">.</span><span class="tok-n">toArray</span>
  <span class="tok-c1">//获取用户已经评分过的商品</span>
  <span class="tok-k">val</span> <span class="tok-n">ratingExist</span> <span class="tok-k">=</span> <span class="tok-nc">ConnHelper</span><span class="tok-o">.</span><span class="tok-n">mongoClient</span><span class="tok-o">(</span><span class="tok-n">mongConfig</span><span class="tok-o">.</span><span class="tok-n">db</span><span class="tok-o">)(</span><span class="tok-nc">MONGODB_RATING_COLLECTION</span><span class="tok-o">).</span><span class="tok-n">find</span><span class="tok-o">(</span><span class="tok-nc">MongoDBObject</span><span class="tok-o">(</span><span class="tok-s">&quot;userId&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-n">userId</span><span class="tok-o">)).</span><span class="tok-n">toArray</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">{</span><span class="tok-n">item</span> <span class="tok-k">=&gt;</span>
    <span class="tok-n">item</span><span class="tok-o">.</span><span class="tok-n">get</span><span class="tok-o">(</span><span class="tok-s">&quot;productId&quot;</span><span class="tok-o">).</span><span class="tok-n">toString</span><span class="tok-o">.</span><span class="tok-n">toInt</span>
  <span class="tok-o">}</span>
  <span class="tok-c1">//过滤掉已经评分过的商品，并排序输出</span>
  <span class="tok-n">allSimProducts</span><span class="tok-o">.</span><span class="tok-n">filter</span><span class="tok-o">(</span><span class="tok-n">x</span> <span class="tok-k">=&gt;</span> <span class="tok-o">!</span><span class="tok-n">ratingExist</span><span class="tok-o">.</span><span class="tok-n">contains</span><span class="tok-o">(</span><span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">)).</span><span class="tok-n">sortWith</span><span class="tok-o">(</span><span class="tok-k">_</span><span class="tok-o">.</span><span class="tok-n">_2</span> <span class="tok-o">&gt;</span> <span class="tok-k">_</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">).</span><span class="tok-n">take</span><span class="tok-o">(</span><span class="tok-n">num</span><span class="tok-o">).</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">x</span> <span class="tok-k">=&gt;</span> <span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">)</span>
<span class="tok-o">}</span></code></pre>
</div>
</div>
</div>
<div class="sect4">
<h5 id="_商品推荐优先级计算">商品推荐优先级计算</h5>
<div class="paragraph">
<p>对于候选商品集合simiHash和userId的最近K个评分recentRatings，算法代码内容如下：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="scala"><span></span><span class="tok-cm">/**</span>
<span class="tok-cm">  * 计算待选商品的推荐分数</span>
<span class="tok-cm">  * @param simProducts            商品相似度矩阵</span>
<span class="tok-cm">  * @param userRecentlyRatings  用户最近的k次评分</span>
<span class="tok-cm">  * @param topSimProducts         当前商品最相似的K个商品</span>
<span class="tok-cm">  * @return</span>
<span class="tok-cm">  */</span>

  <span class="tok-k">def</span> <span class="tok-n">computeProductScores</span><span class="tok-o">(</span><span class="tok-n">simProducts</span><span class="tok-k">:</span> <span class="tok-kt">scala.collection.Map</span><span class="tok-o">[</span><span class="tok-kt">Int</span>,<span class="tok-kt">scala.collection.immutable.Map</span><span class="tok-o">[</span><span class="tok-kt">Int</span>,<span class="tok-kt">Double</span><span class="tok-o">]],</span><span class="tok-n">userRecentlyRatings</span><span class="tok-k">:</span><span class="tok-kt">Array</span><span class="tok-o">[(</span><span class="tok-kt">Int</span>,<span class="tok-kt">Double</span><span class="tok-o">)],</span><span class="tok-n">topSimProducts</span><span class="tok-k">:</span> <span class="tok-kt">Array</span><span class="tok-o">[</span><span class="tok-kt">Int</span><span class="tok-o">])</span><span class="tok-k">:</span> <span class="tok-kt">Array</span><span class="tok-o">[(</span><span class="tok-kt">Int</span>,<span class="tok-kt">Double</span><span class="tok-o">)]</span> <span class="tok-k">=</span> <span class="tok-o">{</span>

    <span class="tok-c1">//用于保存每一个待选商品和最近评分的每一个商品的权重得分</span>
    <span class="tok-k">val</span> <span class="tok-n">score</span> <span class="tok-k">=</span> <span class="tok-n">scala</span><span class="tok-o">.</span><span class="tok-n">collection</span><span class="tok-o">.</span><span class="tok-n">mutable</span><span class="tok-o">.</span><span class="tok-nc">ArrayBuffer</span><span class="tok-o">[(</span><span class="tok-kt">Int</span>,<span class="tok-kt">Double</span><span class="tok-o">)]()</span>

    <span class="tok-c1">//用于保存每一个商品的增强因子数</span>
    <span class="tok-k">val</span> <span class="tok-n">increMap</span> <span class="tok-k">=</span> <span class="tok-n">scala</span><span class="tok-o">.</span><span class="tok-n">collection</span><span class="tok-o">.</span><span class="tok-n">mutable</span><span class="tok-o">.</span><span class="tok-nc">HashMap</span><span class="tok-o">[</span><span class="tok-kt">Int</span>,<span class="tok-kt">Int</span><span class="tok-o">]()</span>

    <span class="tok-c1">//用于保存每一个商品的减弱因子数</span>
    <span class="tok-k">val</span> <span class="tok-n">decreMap</span> <span class="tok-k">=</span> <span class="tok-n">scala</span><span class="tok-o">.</span><span class="tok-n">collection</span><span class="tok-o">.</span><span class="tok-n">mutable</span><span class="tok-o">.</span><span class="tok-nc">HashMap</span><span class="tok-o">[</span><span class="tok-kt">Int</span>,<span class="tok-kt">Int</span><span class="tok-o">]()</span>

    <span class="tok-k">for</span> <span class="tok-o">(</span><span class="tok-n">topSimProduct</span> <span class="tok-k">&lt;-</span> <span class="tok-n">topSimProducts</span><span class="tok-o">;</span> <span class="tok-n">userRecentlyRating</span> <span class="tok-k">&lt;-</span> <span class="tok-n">userRecentlyRatings</span><span class="tok-o">){</span>
      <span class="tok-k">val</span> <span class="tok-n">simScore</span> <span class="tok-k">=</span> <span class="tok-n">getProductsSimScore</span><span class="tok-o">(</span><span class="tok-n">simProducts</span><span class="tok-o">,</span> <span class="tok-n">userRecentlyRating</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">,</span> <span class="tok-n">topSimProduct</span><span class="tok-o">)</span>
      <span class="tok-k">if</span><span class="tok-o">(</span><span class="tok-n">simScore</span> <span class="tok-o">&gt;</span> <span class="tok-mf">0.6</span><span class="tok-o">){</span>
        <span class="tok-n">score</span> <span class="tok-o">+=</span> <span class="tok-o">((</span><span class="tok-n">topSimProduct</span><span class="tok-o">,</span> <span class="tok-n">simScore</span> <span class="tok-o">*</span> <span class="tok-n">userRecentlyRating</span><span class="tok-o">.</span><span class="tok-n">_2</span> <span class="tok-o">))</span>
        <span class="tok-k">if</span><span class="tok-o">(</span><span class="tok-n">userRecentlyRating</span><span class="tok-o">.</span><span class="tok-n">_2</span> <span class="tok-o">&gt;</span> <span class="tok-mi">3</span><span class="tok-o">){</span>
          <span class="tok-n">increMap</span><span class="tok-o">(</span><span class="tok-n">topSimProduct</span><span class="tok-o">)</span> <span class="tok-k">=</span> <span class="tok-n">increMap</span><span class="tok-o">.</span><span class="tok-n">getOrDefault</span><span class="tok-o">(</span><span class="tok-n">topSimProduct</span><span class="tok-o">,</span><span class="tok-mi">0</span><span class="tok-o">)</span> <span class="tok-o">+</span> <span class="tok-mi">1</span>
        <span class="tok-o">}</span><span class="tok-k">else</span><span class="tok-o">{</span>
          <span class="tok-n">decreMap</span><span class="tok-o">(</span><span class="tok-n">topSimProduct</span><span class="tok-o">)</span> <span class="tok-k">=</span> <span class="tok-n">decreMap</span><span class="tok-o">.</span><span class="tok-n">getOrDefault</span><span class="tok-o">(</span><span class="tok-n">topSimProduct</span><span class="tok-o">,</span><span class="tok-mi">0</span><span class="tok-o">)</span> <span class="tok-o">+</span> <span class="tok-mi">1</span>
        <span class="tok-o">}</span>
      <span class="tok-o">}</span>
    <span class="tok-o">}</span>
    <span class="tok-n">score</span><span class="tok-o">.</span><span class="tok-n">groupBy</span><span class="tok-o">(</span><span class="tok-k">_</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">).</span><span class="tok-n">map</span><span class="tok-o">{</span><span class="tok-k">case</span> <span class="tok-o">(</span><span class="tok-n">mid</span><span class="tok-o">,</span><span class="tok-n">sims</span><span class="tok-o">)</span> <span class="tok-k">=&gt;</span>
      <span class="tok-o">(</span><span class="tok-n">mid</span><span class="tok-o">,</span><span class="tok-n">sims</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-k">_</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">).</span><span class="tok-n">sum</span> <span class="tok-o">/</span> <span class="tok-n">sims</span><span class="tok-o">.</span><span class="tok-n">length</span> <span class="tok-o">+</span> <span class="tok-n">log</span><span class="tok-o">(</span><span class="tok-n">increMap</span><span class="tok-o">.</span><span class="tok-n">getOrDefault</span><span class="tok-o">(</span><span class="tok-n">mid</span><span class="tok-o">,</span> <span class="tok-mi">1</span><span class="tok-o">))</span> <span class="tok-o">-</span> <span class="tok-n">log</span><span class="tok-o">(</span><span class="tok-n">decreMap</span><span class="tok-o">.</span><span class="tok-n">getOrDefault</span><span class="tok-o">(</span><span class="tok-n">mid</span><span class="tok-o">,</span> <span class="tok-mi">1</span><span class="tok-o">)))</span>
    <span class="tok-o">}.</span><span class="tok-n">toArray</span>
  <span class="tok-o">}</span></code></pre>
</div>
</div>
<div class="paragraph">
<p>其中，getProductSimScore是取候选商品和已评分商品的相似度，代码如下：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="scala"><span></span><span class="tok-cm">/**</span>
<span class="tok-cm">  * 获取当个商品之间的相似度</span>
<span class="tok-cm">  * @param simProducts       商品相似度矩阵</span>
<span class="tok-cm">  * @param userRatingProduct 用户已经评分的商品</span>
<span class="tok-cm">  * @param topSimProduct     候选商品</span>
<span class="tok-cm">  * @return</span>
<span class="tok-cm">  */</span>
<span class="tok-k">def</span> <span class="tok-n">getProductsSimScore</span><span class="tok-o">(</span><span class="tok-n">simProducts</span><span class="tok-k">:</span> <span class="tok-kt">scala.collection.Map</span><span class="tok-o">[</span><span class="tok-kt">Int</span>,<span class="tok-kt">scala.collection.immutable.Map</span><span class="tok-o">[</span><span class="tok-kt">Int</span>,<span class="tok-kt">Double</span><span class="tok-o">]],</span> <span class="tok-n">userRatingProduct</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">topSimProduct</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">)</span><span class="tok-k">:</span> <span class="tok-kt">Double</span> <span class="tok-o">=</span> <span class="tok-o">{</span>
  <span class="tok-n">simProducts</span><span class="tok-o">.</span><span class="tok-n">get</span><span class="tok-o">(</span><span class="tok-n">topSimProduct</span><span class="tok-o">)</span> <span class="tok-k">match</span> <span class="tok-o">{</span>
    <span class="tok-k">case</span> <span class="tok-nc">Some</span><span class="tok-o">(</span><span class="tok-n">sim</span><span class="tok-o">)</span> <span class="tok-k">=&gt;</span> <span class="tok-n">sim</span><span class="tok-o">.</span><span class="tok-n">get</span><span class="tok-o">(</span><span class="tok-n">userRatingProduct</span><span class="tok-o">)</span> <span class="tok-k">match</span> <span class="tok-o">{</span>
      <span class="tok-k">case</span> <span class="tok-nc">Some</span><span class="tok-o">(</span><span class="tok-n">score</span><span class="tok-o">)</span> <span class="tok-k">=&gt;</span> <span class="tok-n">score</span>
      <span class="tok-k">case</span> <span class="tok-nc">None</span> <span class="tok-k">=&gt;</span> <span class="tok-mf">0.0</span>
    <span class="tok-o">}</span>
    <span class="tok-k">case</span> <span class="tok-nc">None</span> <span class="tok-k">=&gt;</span> <span class="tok-mf">0.0</span>
  <span class="tok-o">}</span>
<span class="tok-o">}</span></code></pre>
</div>
</div>
<div class="paragraph">
<p>而log是对数运算，这里实现为取2的对数（常用对数）：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="scala"><span></span>  <span class="tok-c1">//取2的对数</span>
  <span class="tok-k">def</span> <span class="tok-n">log</span><span class="tok-o">(</span><span class="tok-n">m</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">)</span><span class="tok-k">:</span> <span class="tok-kt">Double</span> <span class="tok-o">=</span> <span class="tok-o">{</span>
    <span class="tok-n">math</span><span class="tok-o">.</span><span class="tok-n">log</span><span class="tok-o">(</span><span class="tok-n">m</span><span class="tok-o">)</span> <span class="tok-o">/</span> <span class="tok-n">math</span><span class="tok-o">.</span><span class="tok-n">log</span><span class="tok-o">(</span><span class="tok-mi">2</span><span class="tok-o">)</span>
  <span class="tok-o">}</span></code></pre>
</div>
</div>
</div>
<div class="sect4">
<h5 id="_将结果保存到mongodb">将结果保存到mongoDB</h5>
<div class="paragraph">
<p>saveRecsToMongoDB函数实现了结果的保存：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="scala"><span></span><span class="tok-cm">/**</span>
<span class="tok-cm">  * 将数据保存到MongoDB    userId -&gt; 1,  recs -&gt; 22:4.5|45:3.8</span>
<span class="tok-cm">  * @param streamRecs  流式的推荐结果</span>
<span class="tok-cm">  * @param mongConfig  MongoDB的配置</span>
<span class="tok-cm">  */</span>
  <span class="tok-k">def</span> <span class="tok-n">saveRecsToMongoDB</span><span class="tok-o">(</span><span class="tok-n">userId</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">streamRecs</span><span class="tok-k">:</span> <span class="tok-kt">Array</span><span class="tok-o">[(</span><span class="tok-kt">Int</span>,<span class="tok-kt">Double</span><span class="tok-o">)])(</span><span class="tok-k">implicit</span> <span class="tok-n">mongConfig</span><span class="tok-k">:</span> <span class="tok-kt">MongConfig</span><span class="tok-o">)</span><span class="tok-k">:</span> <span class="tok-kt">Unit</span> <span class="tok-o">=</span> <span class="tok-o">{</span>
    <span class="tok-c1">//到StreamRecs的连接</span>
    <span class="tok-k">val</span> <span class="tok-n">streamRecsCollection</span> <span class="tok-k">=</span> <span class="tok-nc">ConnHelper</span><span class="tok-o">.</span><span class="tok-n">mongoClient</span><span class="tok-o">(</span><span class="tok-n">mongConfig</span><span class="tok-o">.</span><span class="tok-n">db</span><span class="tok-o">)(</span><span class="tok-nc">MONGODB_STREAM_RECS_COLLECTION</span><span class="tok-o">)</span>

    <span class="tok-n">streamRecsCollection</span><span class="tok-o">.</span><span class="tok-n">findAndRemove</span><span class="tok-o">(</span><span class="tok-nc">MongoDBObject</span><span class="tok-o">(</span><span class="tok-s">&quot;userId&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-n">userId</span><span class="tok-o">))</span>
    <span class="tok-c1">//streaRecsCollection.insert(MongoDBObject(&quot;userId&quot; -&gt; userId, &quot;recs&quot; -&gt; streamRecs.map(x=&gt; x._1+&quot;:&quot;+x._2).mkString(&quot;|&quot;)))</span>
    <span class="tok-n">streamRecsCollection</span><span class="tok-o">.</span><span class="tok-n">insert</span><span class="tok-o">(</span><span class="tok-nc">MongoDBObject</span><span class="tok-o">(</span><span class="tok-s">&quot;userId&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-n">userId</span><span class="tok-o">,</span> <span class="tok-s">&quot;recs&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-n">streamRecs</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">x</span> <span class="tok-k">=&gt;</span> <span class="tok-nc">MongoDBObject</span><span class="tok-o">(</span><span class="tok-s">&quot;productId&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">,</span> <span class="tok-s">&quot;score&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">))))</span>

  <span class="tok-o">}</span></code></pre>
</div>
</div>
</div>
<div class="sect4">
<h5 id="_更新实时推荐结果">更新实时推荐结果</h5>
<div class="paragraph">
<p>当计算出候选商品的推荐优先级的数组updatedRecommends&lt;productId, E&gt;后，这个数组将被发送到Web后台服务器，与后台服务器上userId的上次实时推荐结果recentRecommends&lt;productId, E&gt;进行合并、替换并选出优先级E前K大的商品作为本次新的实时推荐。具体而言：</p>
</div>
<div class="olist loweralpha">
<ol class="loweralpha">
<li>
<p>合并：将updatedRecommends与recentRecommends并集合成为一个新的&lt;productId, E&gt;数组；</p>
</li>
<li>
<p>替换（去重）：当updatedRecommends与recentRecommends有重复的商品productId时，recentRecommends中productId的推荐优先级由于是上次实时推荐的结果，于是将作废，被替换成代表了更新后的updatedRecommends的productId的推荐优先级；</p>
</li>
<li>
<p>选取TopK：在合并、替换后的&lt;ProductId, E&gt;数组上，根据每个product的推荐优先级，选择出前K大的商品，作为本次实时推荐的最终结果。</p>
</li>
</ol>
</div>
</div>
</div>
<div class="sect3">
<h4 id="_实时系统联调">5.9.4. 实时系统联调</h4>
<div class="paragraph">
<p>我们的系统实时推荐的数据流向是：业务系统 &#8594; 日志 &#8594; flume 日志采集 &#8594; kafka streaming数据清洗和预处理 &#8594; spark streaming 流式计算。在我们完成实时推荐服务的代码后，应该与其它工具进行联调测试，确保系统正常运行。</p>
</div>
<div class="sect4">
<h5 id="_启动实时系统的基本组件">启动实时系统的基本组件</h5>
<div class="paragraph">
<p>启动实时推荐系统StreamingRecommender以及mongodb、redis</p>
</div>
</div>
<div class="sect4">
<h5 id="_启动zookeeper">启动zookeeper</h5>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="sh"><span></span>$ bin/zkServer.sh start</code></pre>
</div>
</div>
</div>
<div class="sect4">
<h5 id="_启动kafka">启动kafka</h5>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="sh"><span></span>bin/kafka-server-start.sh -daemon ./config/server.properties</code></pre>
</div>
</div>
</div>
<div class="sect4">
<h5 id="_构建kafka_streaming程序">构建Kafka Streaming程序</h5>
<div class="paragraph">
<p>在recommender下新建module，KafkaStreaming，主要用来做日志数据的预处理，过滤出需要的内容。pom.xml文件需要引入依赖：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="xml"><span></span>    <span class="tok-nt">&lt;dependencies&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.kafka<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>kafka-streams<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>0.10.2.1<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.kafka<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>kafka-clients<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>0.10.2.1<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
    <span class="tok-nt">&lt;/dependencies&gt;</span>

    <span class="tok-nt">&lt;build&gt;</span>
        <span class="tok-nt">&lt;finalName&gt;</span>kafkastream<span class="tok-nt">&lt;/finalName&gt;</span>
        <span class="tok-nt">&lt;plugins&gt;</span>
            <span class="tok-nt">&lt;plugin&gt;</span>
                <span class="tok-nt">&lt;groupId&gt;</span>org.apache.maven.plugins<span class="tok-nt">&lt;/groupId&gt;</span>
                <span class="tok-nt">&lt;artifactId&gt;</span>maven-assembly-plugin<span class="tok-nt">&lt;/artifactId&gt;</span>
                <span class="tok-nt">&lt;configuration&gt;</span>
                    <span class="tok-nt">&lt;archive&gt;</span>
                        <span class="tok-nt">&lt;manifest&gt;</span>
                            <span class="tok-nt">&lt;mainClass&gt;</span>com.atguigu.kafkastream.Application<span class="tok-nt">&lt;/mainClass&gt;</span>
                        <span class="tok-nt">&lt;/manifest&gt;</span>
                    <span class="tok-nt">&lt;/archive&gt;</span>
                    <span class="tok-nt">&lt;descriptorRefs&gt;</span>
                        <span class="tok-nt">&lt;descriptorRef&gt;</span>jar-with-dependencies<span class="tok-nt">&lt;/descriptorRef&gt;</span>
                    <span class="tok-nt">&lt;/descriptorRefs&gt;</span>
                <span class="tok-nt">&lt;/configuration&gt;</span>
                <span class="tok-nt">&lt;executions&gt;</span>
                    <span class="tok-nt">&lt;execution&gt;</span>
                        <span class="tok-nt">&lt;id&gt;</span>make-assembly<span class="tok-nt">&lt;/id&gt;</span>
                        <span class="tok-nt">&lt;phase&gt;</span>package<span class="tok-nt">&lt;/phase&gt;</span>
                        <span class="tok-nt">&lt;goals&gt;</span>
                            <span class="tok-nt">&lt;goal&gt;</span>single<span class="tok-nt">&lt;/goal&gt;</span>
                        <span class="tok-nt">&lt;/goals&gt;</span>
                    <span class="tok-nt">&lt;/execution&gt;</span>
                <span class="tok-nt">&lt;/executions&gt;</span>
            <span class="tok-nt">&lt;/plugin&gt;</span>
        <span class="tok-nt">&lt;/plugins&gt;</span>
    <span class="tok-nt">&lt;/build&gt;</span></code></pre>
</div>
</div>
<div class="paragraph">
<p>在src/main/java下新建java类com.atguigu.kafkastreaming.Application</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="java"><span></span><span class="tok-kn">package</span> <span class="tok-nn">com.atguigu.kafkastream</span><span class="tok-o">;</span>

<span class="tok-kn">import</span> <span class="tok-nn">org.apache.kafka.streams.KafkaStreams</span><span class="tok-o">;</span>
<span class="tok-kn">import</span> <span class="tok-nn">org.apache.kafka.streams.StreamsConfig</span><span class="tok-o">;</span>
<span class="tok-kn">import</span> <span class="tok-nn">org.apache.kafka.streams.processor.TopologyBuilder</span><span class="tok-o">;</span>

<span class="tok-kn">import</span> <span class="tok-nn">java.util.Properties</span><span class="tok-o">;</span>

<span class="tok-kd">public</span> <span class="tok-kd">class</span> <span class="tok-nc">Application</span> <span class="tok-o">{</span>
    <span class="tok-kd">public</span> <span class="tok-kd">static</span> <span class="tok-kt">void</span> <span class="tok-nf">main</span><span class="tok-o">(</span><span class="tok-n">String</span><span class="tok-o">[]</span> <span class="tok-n">args</span><span class="tok-o">){</span>
        <span class="tok-n">String</span> <span class="tok-n">brokers</span> <span class="tok-o">=</span> <span class="tok-s">&quot;localhost:9092&quot;</span><span class="tok-o">;</span>
        <span class="tok-n">String</span> <span class="tok-n">zookeepers</span> <span class="tok-o">=</span> <span class="tok-s">&quot;localhost:2181&quot;</span><span class="tok-o">;</span>
        <span class="tok-n">String</span> <span class="tok-n">from</span> <span class="tok-o">=</span> <span class="tok-s">&quot;log&quot;</span><span class="tok-o">;</span>
        <span class="tok-n">String</span> <span class="tok-n">to</span> <span class="tok-o">=</span> <span class="tok-s">&quot;recommender&quot;</span><span class="tok-o">;</span>

        <span class="tok-n">Properties</span> <span class="tok-n">settings</span> <span class="tok-o">=</span> <span class="tok-k">new</span> <span class="tok-n">Properties</span><span class="tok-o">();</span>
        <span class="tok-n">settings</span><span class="tok-o">.</span><span class="tok-na">put</span><span class="tok-o">(</span><span class="tok-n">StreamsConfig</span><span class="tok-o">.</span><span class="tok-na">APPLICATION_ID_CONFIG</span><span class="tok-o">,</span> <span class="tok-s">&quot;logFilter&quot;</span><span class="tok-o">);</span>
        <span class="tok-n">settings</span><span class="tok-o">.</span><span class="tok-na">put</span><span class="tok-o">(</span><span class="tok-n">StreamsConfig</span><span class="tok-o">.</span><span class="tok-na">BOOTSTRAP_SERVERS_CONFIG</span><span class="tok-o">,</span> <span class="tok-n">brokers</span><span class="tok-o">);</span>
        <span class="tok-n">settings</span><span class="tok-o">.</span><span class="tok-na">put</span><span class="tok-o">(</span><span class="tok-n">StreamsConfig</span><span class="tok-o">.</span><span class="tok-na">ZOOKEEPER_CONNECT_CONFIG</span><span class="tok-o">,</span> <span class="tok-n">zookeepers</span><span class="tok-o">);</span>

        <span class="tok-n">StreamsConfig</span> <span class="tok-n">config</span> <span class="tok-o">=</span> <span class="tok-k">new</span> <span class="tok-n">StreamsConfig</span><span class="tok-o">(</span><span class="tok-n">settings</span><span class="tok-o">);</span>

        <span class="tok-n">TopologyBuilder</span> <span class="tok-n">builder</span> <span class="tok-o">=</span> <span class="tok-k">new</span> <span class="tok-n">TopologyBuilder</span><span class="tok-o">();</span>

        <span class="tok-n">builder</span><span class="tok-o">.</span><span class="tok-na">addSource</span><span class="tok-o">(</span><span class="tok-s">&quot;SOURCE&quot;</span><span class="tok-o">,</span> <span class="tok-n">from</span><span class="tok-o">)</span>
                <span class="tok-o">.</span><span class="tok-na">addProcessor</span><span class="tok-o">(</span><span class="tok-s">&quot;PROCESS&quot;</span><span class="tok-o">,</span> <span class="tok-o">()</span> <span class="tok-o">-&gt;</span> <span class="tok-k">new</span> <span class="tok-n">LogProcessor</span><span class="tok-o">(),</span> <span class="tok-s">&quot;SOURCE&quot;</span><span class="tok-o">)</span>
                <span class="tok-o">.</span><span class="tok-na">addSink</span><span class="tok-o">(</span><span class="tok-s">&quot;SINK&quot;</span><span class="tok-o">,</span> <span class="tok-n">to</span><span class="tok-o">,</span> <span class="tok-s">&quot;PROCESS&quot;</span><span class="tok-o">);</span>

        <span class="tok-n">KafkaStreams</span> <span class="tok-n">streams</span> <span class="tok-o">=</span> <span class="tok-k">new</span> <span class="tok-n">KafkaStreams</span><span class="tok-o">(</span><span class="tok-n">builder</span><span class="tok-o">,</span> <span class="tok-n">config</span><span class="tok-o">);</span>
        <span class="tok-n">streams</span><span class="tok-o">.</span><span class="tok-na">start</span><span class="tok-o">();</span>
    <span class="tok-o">}</span>
<span class="tok-o">}</span></code></pre>
</div>
</div>
<div class="paragraph">
<p>这个程序会将topic为“log”的信息流获取来做处理，并以“recommender”为新的topic转发出去。</p>
</div>
<div class="paragraph">
<p>流处理程序LogProcess.java</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="java"><span></span><span class="tok-kn">package</span> <span class="tok-nn">com.atguigu.kafkastream</span><span class="tok-o">;</span>

<span class="tok-kn">import</span> <span class="tok-nn">org.apache.kafka.streams.processor.Processor</span><span class="tok-o">;</span>
<span class="tok-kn">import</span> <span class="tok-nn">org.apache.kafka.streams.processor.ProcessorContext</span><span class="tok-o">;</span>

<span class="tok-kd">public</span> <span class="tok-kd">class</span> <span class="tok-nc">LogProcessor</span> <span class="tok-kd">implements</span> <span class="tok-n">Processor</span><span class="tok-o">&lt;</span><span class="tok-kt">byte</span><span class="tok-o">[],</span><span class="tok-kt">byte</span><span class="tok-o">[]&gt;</span> <span class="tok-o">{</span>
    <span class="tok-kd">private</span> <span class="tok-n">ProcessorContext</span> <span class="tok-n">context</span><span class="tok-o">;</span>

    <span class="tok-kd">public</span> <span class="tok-kt">void</span> <span class="tok-nf">init</span><span class="tok-o">(</span><span class="tok-n">ProcessorContext</span> <span class="tok-n">context</span><span class="tok-o">)</span> <span class="tok-o">{</span>
        <span class="tok-k">this</span><span class="tok-o">.</span><span class="tok-na">context</span> <span class="tok-o">=</span> <span class="tok-n">context</span><span class="tok-o">;</span>
    <span class="tok-o">}</span>

    <span class="tok-kd">public</span> <span class="tok-kt">void</span> <span class="tok-nf">process</span><span class="tok-o">(</span><span class="tok-kt">byte</span><span class="tok-o">[]</span> <span class="tok-n">dummy</span><span class="tok-o">,</span> <span class="tok-kt">byte</span><span class="tok-o">[]</span> <span class="tok-n">line</span><span class="tok-o">)</span> <span class="tok-o">{</span>
        <span class="tok-n">String</span> <span class="tok-n">input</span> <span class="tok-o">=</span> <span class="tok-k">new</span> <span class="tok-n">String</span><span class="tok-o">(</span><span class="tok-n">line</span><span class="tok-o">);</span>
        <span class="tok-k">if</span><span class="tok-o">(</span><span class="tok-n">input</span><span class="tok-o">.</span><span class="tok-na">contains</span><span class="tok-o">(</span><span class="tok-s">&quot;PRODUCT_RATING_PREFIX:&quot;</span><span class="tok-o">)){</span>
            <span class="tok-n">System</span><span class="tok-o">.</span><span class="tok-na">out</span><span class="tok-o">.</span><span class="tok-na">println</span><span class="tok-o">(</span><span class="tok-s">&quot;product rating coming!!!!&quot;</span><span class="tok-o">);</span>
            <span class="tok-n">input</span> <span class="tok-o">=</span> <span class="tok-n">input</span><span class="tok-o">.</span><span class="tok-na">split</span><span class="tok-o">(</span><span class="tok-s">&quot;PRODUCT_RATING_PREFIX:&quot;</span><span class="tok-o">)[</span><span class="tok-mi">1</span><span class="tok-o">].</span><span class="tok-na">trim</span><span class="tok-o">();</span>
            <span class="tok-n">context</span><span class="tok-o">.</span><span class="tok-na">forward</span><span class="tok-o">(</span><span class="tok-s">&quot;logProcessor&quot;</span><span class="tok-o">.</span><span class="tok-na">getBytes</span><span class="tok-o">(),</span> <span class="tok-n">input</span><span class="tok-o">.</span><span class="tok-na">getBytes</span><span class="tok-o">());</span>
        <span class="tok-o">}</span>
    <span class="tok-o">}</span>

    <span class="tok-kd">public</span> <span class="tok-kt">void</span> <span class="tok-nf">punctuate</span><span class="tok-o">(</span><span class="tok-kt">long</span> <span class="tok-n">timestamp</span><span class="tok-o">)</span> <span class="tok-o">{</span>
    <span class="tok-o">}</span>

    <span class="tok-kd">public</span> <span class="tok-kt">void</span> <span class="tok-nf">close</span><span class="tok-o">()</span> <span class="tok-o">{</span>
    <span class="tok-o">}</span>
<span class="tok-o">}</span></code></pre>
</div>
</div>
<div class="paragraph">
<p>完成代码后，启动Application。</p>
</div>
</div>
<div class="sect4">
<h5 id="_配置并启动flume">配置并启动flume</h5>
<div class="paragraph">
<p>在flume的conf目录下新建log-kafka.properties，对flume连接kafka做配置：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code><span></span>agent.sources = exectail
agent.channels = memoryChannel
agent.sinks = kafkasink

# For each one of the sources, the type is defined
agent.sources.exectail.type = exec
# 下面这个路径是需要收集日志的绝对路径，改为自己的日志目录
agent.sources.exectail.command = tail –f /mnt/d/Projects/BigData/ProductRecommender/businessServer/src/main/log/agent.log
agent.sources.exectail.interceptors = i1

agent.sources.exectail.interceptors.i1.type=regex_filter
# 定义日志过滤前缀的正则
agent.sources.exectail.interceptors.i1.regex=.+PRODUCT_RATING_PREFIX.+
# The channel can be defined as follows.
agent.sources.exectail.channels = memoryChannel

# Each sink&#39;s type must be defined
agent.sinks.kafkasink.type = org.apache.flume.sink.kafka.KafkaSink
agent.sinks.kafkasink.kafka.topic = log
agent.sinks.kafkasink.kafka.bootstrap.servers = localhost:9092
agent.sinks.kafkasink.kafka.producer.acks = 1
agent.sinks.kafkasink.kafka.flumeBatchSize = 20

#Specify the channel the sink should use
agent.sinks.kafkasink.channel = memoryChannel

# Each channel&#39;s type is defined.
agent.channels.memoryChannel.type = memory

# Other config values specific to each type of channel(sink or source)
# can be defined as well
# In this case, it specifies the capacity of the memory channel
agent.channels.memoryChannel.capacity = 10000</code></pre>
</div>
</div>
<div class="paragraph">
<p>配置好后，启动flume：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="sh"><span></span>$ ./bin/flume-ng agent -c ./conf/ -f ./conf/log-kafka.properties -n agent -Dflume.root.logger<span class="tok-o">=</span>INFO,console</code></pre>
</div>
</div>
</div>
<div class="sect4">
<h5 id="_启动业务系统后台">启动业务系统后台</h5>
<div class="paragraph">
<p>将业务代码加入系统中, 将仓库中的`businessServer`文件夹拷贝到项目中, 和`recommender`同级。注意在`src/main/resources/`下的`log4j.properties`中，`log4j.appender.file.File`的值应该替换为自己的日志目录，与flume中的配置应该相同。例如:</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code><span></span>log4j.appender.file.File=/mnt/d/Projects/BigData/ProductRecommender/businessServer/src/main/log/agent.log</code></pre>
</div>
</div>
<div class="paragraph">
<p>运行业务系统:</p>
</div>
<div class="olist arabic">
<ol class="arabic">
<li>
<p>点击idea右侧的maven projects.</p>
</li>
<li>
<p>找到businessServer中的plugins中的tomcat7插件.</p>
</li>
<li>
<p>双击运行tomcat7:run.</p>
</li>
</ol>
</div>
<div class="paragraph">
<p>启动业务系统后台，访问`localhost:8088`；点击某个商品进行评分，查看实时推荐列表是否会发生变化。</p>
</div>
</div>
</div>
</div>
<div class="sect2">
<h3 id="_冷启动问题处理">5.10. 冷启动问题处理</h3>
<div class="paragraph">
<p>整个推荐系统更多的是依赖于用于的偏好信息进行商品的推荐，那么就会存在一个问题，对于新注册的用户是没有任何偏好信息记录的，那这个时候推荐就会出现问题，导致没有任何推荐的商品出现。</p>
</div>
<div class="paragraph">
<p>我们在电商推荐中解决冷启动的方案是：给新注册的用户推荐热门的商品，例如近期热门商品、历史热门商品等策略。</p>
</div>
<div class="paragraph">
<p>实际生产环境中可以根据不同的业务场景调整策略。例如资讯类应用可以先给用户一个交互式的标签页面，让用户自己选择感兴趣的标签，然后推荐用户感兴趣标签的热门内容。</p>
</div>
</div>
<div class="sect2">
<h3 id="_基于内容的推荐服务">5.11. 基于内容的推荐服务</h3>
<div class="sect3">
<h4 id="_基于内容的推荐服务_2">5.11.1. 基于内容的推荐服务</h4>
<div class="paragraph">
<p>原始数据中的tags字段，是用户给商品打上的标签，这部分内容想要直接转成评分并不容易，不过我们可以将标签内容进行提取，得到商品的内容特征向量，进而可以通过求取相似度矩阵。这部分可以与实时推荐系统直接对接，计算出与用户当前评分商品的相似商品，实现基于内容的实时推荐。为了避免热门标签对特征提取的影响，我们还可以通过TF-IDF算法对标签的权重进行调整，从而尽可能地接近用户偏好。</p>
</div>
</div>
<div class="sect3">
<h4 id="_基于内容推荐的实现">5.11.2. 基于内容推荐的实现</h4>
<div class="paragraph">
<p>基于以上思想，加入TF-IDF算法的求取商品特征向量的核心代码如下：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="scala"><span></span><span class="tok-k">package</span> <span class="tok-nn">com.atguigu.content</span>

<span class="tok-k">import</span> <span class="tok-nn">org.apache.spark.SparkConf</span>
<span class="tok-k">import</span> <span class="tok-nn">org.apache.spark.sql.SparkSession</span>
<span class="tok-k">import</span> <span class="tok-nn">org.apache.spark.ml.feature.</span><span class="tok-o">{</span><span class="tok-nc">HashingTF</span><span class="tok-o">,</span> <span class="tok-nc">IDF</span><span class="tok-o">,</span> <span class="tok-nc">Tokenizer</span><span class="tok-o">}</span>
<span class="tok-k">import</span> <span class="tok-nn">org.apache.spark.ml.linalg.SparseVector</span>
<span class="tok-k">import</span> <span class="tok-nn">org.jblas.DoubleMatrix</span>

<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">MongoConfig</span><span class="tok-o">(</span><span class="tok-n">uri</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">,</span> <span class="tok-n">db</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">)</span>

<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">Product</span><span class="tok-o">(</span><span class="tok-n">productId</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">name</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">,</span> <span class="tok-n">categories</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">,</span> <span class="tok-n">imageUrl</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">,</span> <span class="tok-n">tags</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">)</span>
<span class="tok-c1">//推荐</span>
<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">Recommendation</span><span class="tok-o">(</span><span class="tok-n">rid</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">r</span><span class="tok-k">:</span> <span class="tok-kt">Double</span><span class="tok-o">)</span>

<span class="tok-c1">// 用户的推荐</span>
<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">UserRecs</span><span class="tok-o">(</span><span class="tok-n">userId</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">recs</span><span class="tok-k">:</span> <span class="tok-kt">Seq</span><span class="tok-o">[</span><span class="tok-kt">Recommendation</span><span class="tok-o">])</span>

<span class="tok-c1">//商品的相似度</span>
<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">ProductRecs</span><span class="tok-o">(</span><span class="tok-n">productId</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">recs</span><span class="tok-k">:</span> <span class="tok-kt">Seq</span><span class="tok-o">[</span><span class="tok-kt">Recommendation</span><span class="tok-o">])</span>

<span class="tok-k">object</span> <span class="tok-nc">ContentBasedRecommender</span> <span class="tok-o">{</span>
  <span class="tok-k">val</span> <span class="tok-nc">MONGODB_PRODUCT_COLLECTION</span> <span class="tok-k">=</span> <span class="tok-s">&quot;Products&quot;</span>
  <span class="tok-k">val</span> <span class="tok-nc">PRODUCT_RECS</span> <span class="tok-k">=</span> <span class="tok-s">&quot;ContentBasedProductRecs&quot;</span>

  <span class="tok-k">def</span> <span class="tok-n">consinSim</span><span class="tok-o">(</span><span class="tok-n">product1</span><span class="tok-k">:</span> <span class="tok-kt">DoubleMatrix</span><span class="tok-o">,</span> <span class="tok-n">product2</span><span class="tok-k">:</span> <span class="tok-kt">DoubleMatrix</span><span class="tok-o">)</span> <span class="tok-k">:</span> <span class="tok-kt">Double</span> <span class="tok-o">={</span>
    <span class="tok-n">product1</span><span class="tok-o">.</span><span class="tok-n">dot</span><span class="tok-o">(</span><span class="tok-n">product2</span><span class="tok-o">)</span> <span class="tok-o">/</span> <span class="tok-o">(</span> <span class="tok-n">product1</span><span class="tok-o">.</span><span class="tok-n">norm2</span><span class="tok-o">()</span> <span class="tok-o">*</span> <span class="tok-n">product2</span><span class="tok-o">.</span><span class="tok-n">norm2</span><span class="tok-o">()</span> <span class="tok-o">)</span>
  <span class="tok-o">}</span>


  <span class="tok-k">def</span> <span class="tok-n">main</span><span class="tok-o">(</span><span class="tok-n">args</span><span class="tok-k">:</span> <span class="tok-kt">Array</span><span class="tok-o">[</span><span class="tok-kt">String</span><span class="tok-o">])</span><span class="tok-k">:</span> <span class="tok-kt">Unit</span> <span class="tok-o">=</span> <span class="tok-o">{</span>

    <span class="tok-k">val</span> <span class="tok-n">config</span> <span class="tok-k">=</span> <span class="tok-nc">Map</span><span class="tok-o">(</span>
      <span class="tok-s">&quot;spark.cores&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-s">&quot;local[*]&quot;</span><span class="tok-o">,</span>
      <span class="tok-s">&quot;mongo.uri&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-s">&quot;mongodb://localhost:27017/recommender&quot;</span><span class="tok-o">,</span>
      <span class="tok-s">&quot;mongo.db&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-s">&quot;reommender&quot;</span>
    <span class="tok-o">)</span>

    <span class="tok-c1">//创建一个SparkConf配置</span>
    <span class="tok-k">val</span> <span class="tok-n">sparkConf</span> <span class="tok-k">=</span> <span class="tok-k">new</span> <span class="tok-nc">SparkConf</span><span class="tok-o">().</span><span class="tok-n">setAppName</span><span class="tok-o">(</span><span class="tok-s">&quot;ContentBasedRecommender&quot;</span><span class="tok-o">).</span><span class="tok-n">setMaster</span><span class="tok-o">(</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-s">&quot;spark.cores&quot;</span><span class="tok-o">)).</span><span class="tok-n">set</span><span class="tok-o">(</span><span class="tok-s">&quot;spark.executor.memory&quot;</span><span class="tok-o">,</span><span class="tok-s">&quot;6G&quot;</span><span class="tok-o">).</span><span class="tok-n">set</span><span class="tok-o">(</span><span class="tok-s">&quot;spark.driver.memory&quot;</span><span class="tok-o">,</span><span class="tok-s">&quot;2G&quot;</span><span class="tok-o">)</span>

    <span class="tok-c1">//基于SparkConf创建一个SparkSession</span>
    <span class="tok-k">val</span> <span class="tok-n">spark</span> <span class="tok-k">=</span> <span class="tok-nc">SparkSession</span><span class="tok-o">.</span><span class="tok-n">builder</span><span class="tok-o">().</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-n">sparkConf</span><span class="tok-o">).</span><span class="tok-n">getOrCreate</span><span class="tok-o">()</span>

    <span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">sparkContext</span><span class="tok-o">.</span><span class="tok-n">setLogLevel</span><span class="tok-o">(</span><span class="tok-s">&quot;ERROR&quot;</span><span class="tok-o">)</span>

    <span class="tok-c1">//创建一个MongoDBConfig</span>
    <span class="tok-k">val</span> <span class="tok-n">mongoConfig</span> <span class="tok-k">=</span> <span class="tok-nc">MongoConfig</span><span class="tok-o">(</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-s">&quot;mongo.uri&quot;</span><span class="tok-o">),</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-s">&quot;mongo.db&quot;</span><span class="tok-o">))</span>

    <span class="tok-k">import</span> <span class="tok-nn">spark.implicits._</span>

    <span class="tok-k">val</span> <span class="tok-n">productRDD</span> <span class="tok-k">=</span> <span class="tok-n">spark</span>
      <span class="tok-o">.</span><span class="tok-n">read</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;uri&quot;</span><span class="tok-o">,</span><span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">uri</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;collection&quot;</span><span class="tok-o">,</span><span class="tok-nc">MONGODB_PRODUCT_COLLECTION</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">format</span><span class="tok-o">(</span><span class="tok-s">&quot;com.mongodb.spark.sql&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">load</span><span class="tok-o">()</span>
      <span class="tok-o">.</span><span class="tok-n">as</span><span class="tok-o">[</span><span class="tok-kt">Product</span><span class="tok-o">]</span>
      <span class="tok-o">.</span><span class="tok-n">rdd</span>
      <span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">x</span> <span class="tok-k">=&gt;</span> <span class="tok-o">(</span><span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">productId</span><span class="tok-o">,</span> <span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">name</span><span class="tok-o">,</span> <span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">tags</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">c</span> <span class="tok-k">=&gt;</span> <span class="tok-k">if</span><span class="tok-o">(</span><span class="tok-n">c</span> <span class="tok-o">==</span> <span class="tok-sc">&#39;|&#39;</span><span class="tok-o">)</span> <span class="tok-sc">&#39; &#39;</span> <span class="tok-k">else</span> <span class="tok-n">c</span><span class="tok-o">)))</span>

    <span class="tok-k">val</span> <span class="tok-n">productSeq</span> <span class="tok-k">=</span> <span class="tok-n">productRDD</span><span class="tok-o">.</span><span class="tok-n">collect</span><span class="tok-o">()</span>

    <span class="tok-k">val</span> <span class="tok-n">tagsData</span> <span class="tok-k">=</span> <span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">createDataFrame</span><span class="tok-o">(</span><span class="tok-n">productSeq</span><span class="tok-o">).</span><span class="tok-n">toDF</span><span class="tok-o">(</span><span class="tok-s">&quot;productId&quot;</span><span class="tok-o">,</span> <span class="tok-s">&quot;name&quot;</span><span class="tok-o">,</span> <span class="tok-s">&quot;tags&quot;</span><span class="tok-o">)</span>

    <span class="tok-c1">// 实例化一个分词器，默认按空格分</span>
    <span class="tok-k">val</span> <span class="tok-n">tokenizer</span> <span class="tok-k">=</span> <span class="tok-k">new</span> <span class="tok-nc">Tokenizer</span><span class="tok-o">().</span><span class="tok-n">setInputCol</span><span class="tok-o">(</span><span class="tok-s">&quot;tags&quot;</span><span class="tok-o">).</span><span class="tok-n">setOutputCol</span><span class="tok-o">(</span><span class="tok-s">&quot;words&quot;</span><span class="tok-o">)</span>

    <span class="tok-c1">// 用分词器做转换，生成列“words”，返回一个dataframe，增加一列words</span>
    <span class="tok-k">val</span> <span class="tok-n">wordsData</span> <span class="tok-k">=</span> <span class="tok-n">tokenizer</span><span class="tok-o">.</span><span class="tok-n">transform</span><span class="tok-o">(</span><span class="tok-n">tagsData</span><span class="tok-o">)</span>

    <span class="tok-n">wordsData</span><span class="tok-o">.</span><span class="tok-n">show</span><span class="tok-o">(</span><span class="tok-mi">5</span><span class="tok-o">)</span>

    <span class="tok-c1">// HashingTF是一个工具，可以把一个词语序列，转换成词频(初始特征)</span>
    <span class="tok-k">val</span> <span class="tok-n">hashingTF</span> <span class="tok-k">=</span> <span class="tok-k">new</span> <span class="tok-nc">HashingTF</span><span class="tok-o">().</span><span class="tok-n">setInputCol</span><span class="tok-o">(</span><span class="tok-s">&quot;words&quot;</span><span class="tok-o">).</span><span class="tok-n">setOutputCol</span><span class="tok-o">(</span><span class="tok-s">&quot;rawFeatures&quot;</span><span class="tok-o">).</span><span class="tok-n">setNumFeatures</span><span class="tok-o">(</span><span class="tok-mi">189</span><span class="tok-o">)</span>

    <span class="tok-c1">// 用 HashingTF 做处理，返回dataframe</span>
    <span class="tok-k">val</span> <span class="tok-n">featurizedData</span> <span class="tok-k">=</span> <span class="tok-n">hashingTF</span><span class="tok-o">.</span><span class="tok-n">transform</span><span class="tok-o">(</span><span class="tok-n">wordsData</span><span class="tok-o">)</span>

    <span class="tok-c1">// IDF 也是一个工具，用于计算文档的IDF</span>
    <span class="tok-k">val</span> <span class="tok-n">idf</span> <span class="tok-k">=</span> <span class="tok-k">new</span> <span class="tok-nc">IDF</span><span class="tok-o">().</span><span class="tok-n">setInputCol</span><span class="tok-o">(</span><span class="tok-s">&quot;rawFeatures&quot;</span><span class="tok-o">).</span><span class="tok-n">setOutputCol</span><span class="tok-o">(</span><span class="tok-s">&quot;features&quot;</span><span class="tok-o">)</span>

    <span class="tok-c1">// 将词频数据传入，得到idf模型（统计文档）</span>
    <span class="tok-k">val</span> <span class="tok-n">idfModel</span> <span class="tok-k">=</span> <span class="tok-n">idf</span><span class="tok-o">.</span><span class="tok-n">fit</span><span class="tok-o">(</span><span class="tok-n">featurizedData</span><span class="tok-o">)</span>

    <span class="tok-c1">// 模型对原始数据做处理，计算出idf后，用tf-idf得到新的特征矩阵</span>
    <span class="tok-k">val</span> <span class="tok-n">rescaledData</span> <span class="tok-k">=</span> <span class="tok-n">idfModel</span><span class="tok-o">.</span><span class="tok-n">transform</span><span class="tok-o">(</span><span class="tok-n">featurizedData</span><span class="tok-o">)</span>

    <span class="tok-n">rescaledData</span><span class="tok-o">.</span><span class="tok-n">show</span><span class="tok-o">(</span><span class="tok-mi">5</span><span class="tok-o">)</span>

    <span class="tok-k">val</span> <span class="tok-n">productFeatures</span> <span class="tok-k">=</span> <span class="tok-n">rescaledData</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">{</span>
      <span class="tok-k">case</span> <span class="tok-n">row</span> <span class="tok-k">=&gt;</span> <span class="tok-o">{</span>
        <span class="tok-k">if</span> <span class="tok-o">(</span><span class="tok-n">row</span><span class="tok-o">.</span><span class="tok-n">getAs</span><span class="tok-o">[</span><span class="tok-kt">Int</span><span class="tok-o">](</span><span class="tok-s">&quot;productId&quot;</span><span class="tok-o">)</span> <span class="tok-o">==</span> <span class="tok-mi">160597</span> <span class="tok-o">||</span> <span class="tok-n">row</span><span class="tok-o">.</span><span class="tok-n">getAs</span><span class="tok-o">[</span><span class="tok-kt">Int</span><span class="tok-o">](</span><span class="tok-s">&quot;productId&quot;</span><span class="tok-o">)</span> <span class="tok-o">==</span> <span class="tok-mi">8195</span><span class="tok-o">)</span> <span class="tok-o">{</span>
          <span class="tok-n">println</span><span class="tok-o">(</span><span class="tok-n">row</span><span class="tok-o">)</span>
        <span class="tok-o">}</span>
        <span class="tok-o">(</span><span class="tok-n">row</span><span class="tok-o">.</span><span class="tok-n">getAs</span><span class="tok-o">[</span><span class="tok-kt">Int</span><span class="tok-o">](</span><span class="tok-s">&quot;productId&quot;</span><span class="tok-o">),</span> <span class="tok-n">row</span><span class="tok-o">.</span><span class="tok-n">getAs</span><span class="tok-o">[</span><span class="tok-kt">SparseVector</span><span class="tok-o">](</span><span class="tok-s">&quot;features&quot;</span><span class="tok-o">).</span><span class="tok-n">toArray</span><span class="tok-o">)</span>
      <span class="tok-o">}</span>
    <span class="tok-o">}</span>
    <span class="tok-o">.</span><span class="tok-n">rdd</span>
    <span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">x</span> <span class="tok-k">=&gt;</span> <span class="tok-o">{</span>
      <span class="tok-o">(</span><span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">,</span> <span class="tok-k">new</span> <span class="tok-nc">DoubleMatrix</span><span class="tok-o">(</span><span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">)</span> <span class="tok-o">)</span>
    <span class="tok-o">})</span>

    <span class="tok-k">val</span> <span class="tok-n">productRecs</span> <span class="tok-k">=</span> <span class="tok-n">productFeatures</span><span class="tok-o">.</span><span class="tok-n">cartesian</span><span class="tok-o">(</span><span class="tok-n">productFeatures</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">filter</span><span class="tok-o">{</span><span class="tok-k">case</span> <span class="tok-o">(</span><span class="tok-n">a</span><span class="tok-o">,</span> <span class="tok-n">b</span><span class="tok-o">)</span> <span class="tok-k">=&gt;</span> <span class="tok-n">a</span><span class="tok-o">.</span><span class="tok-n">_1</span> <span class="tok-o">!=</span> <span class="tok-n">b</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">}</span>
      <span class="tok-o">.</span><span class="tok-n">map</span> <span class="tok-o">{</span>
        <span class="tok-k">case</span> <span class="tok-o">(</span><span class="tok-n">a</span><span class="tok-o">,</span> <span class="tok-n">b</span><span class="tok-o">)</span> <span class="tok-k">=&gt;</span> <span class="tok-o">{</span>
          <span class="tok-k">if</span> <span class="tok-o">(</span><span class="tok-n">a</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">==</span><span class="tok-mi">160597</span> <span class="tok-o">&amp;&amp;</span> <span class="tok-n">b</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">==</span><span class="tok-mi">8195</span><span class="tok-o">)</span> <span class="tok-o">{</span>
            <span class="tok-n">println</span><span class="tok-o">(</span><span class="tok-n">a</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">,</span> <span class="tok-n">a</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">,</span> <span class="tok-n">b</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">,</span> <span class="tok-n">b</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">)</span>
          <span class="tok-o">}</span>
          <span class="tok-k">val</span> <span class="tok-n">simScore</span> <span class="tok-k">=</span> <span class="tok-k">this</span><span class="tok-o">.</span><span class="tok-n">consinSim</span><span class="tok-o">(</span><span class="tok-n">a</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">,</span> <span class="tok-n">b</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">)</span>
          <span class="tok-o">(</span><span class="tok-n">a</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">,</span> <span class="tok-o">(</span><span class="tok-n">b</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">,</span> <span class="tok-n">simScore</span><span class="tok-o">))</span>
        <span class="tok-o">}</span>
      <span class="tok-o">}</span>
      <span class="tok-o">.</span><span class="tok-n">groupByKey</span><span class="tok-o">()</span>
      <span class="tok-o">.</span><span class="tok-n">map</span> <span class="tok-o">{</span>
        <span class="tok-k">case</span> <span class="tok-o">(</span><span class="tok-n">productId</span><span class="tok-o">,</span> <span class="tok-n">items</span><span class="tok-o">)</span> <span class="tok-k">=&gt;</span> <span class="tok-nc">ProductRecs</span><span class="tok-o">(</span><span class="tok-n">productId</span><span class="tok-o">,</span> <span class="tok-n">items</span><span class="tok-o">.</span><span class="tok-n">toList</span><span class="tok-o">.</span><span class="tok-n">sortWith</span><span class="tok-o">(</span><span class="tok-k">_</span><span class="tok-o">.</span><span class="tok-n">_2</span> <span class="tok-o">&gt;</span> <span class="tok-k">_</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">).</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">x</span> <span class="tok-k">=&gt;</span> <span class="tok-nc">Recommendation</span><span class="tok-o">(</span><span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">,</span> <span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">)).</span><span class="tok-n">take</span><span class="tok-o">(</span><span class="tok-mi">5</span><span class="tok-o">))</span>
      <span class="tok-o">}</span>
      <span class="tok-o">.</span><span class="tok-n">toDF</span><span class="tok-o">()</span>

    <span class="tok-n">productRecs</span><span class="tok-o">.</span><span class="tok-n">show</span><span class="tok-o">(</span><span class="tok-mi">5</span><span class="tok-o">)</span>

    <span class="tok-n">productRecs</span>
      <span class="tok-o">.</span><span class="tok-n">write</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;uri&quot;</span><span class="tok-o">,</span> <span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">uri</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;collection&quot;</span><span class="tok-o">,</span> <span class="tok-nc">PRODUCT_RECS</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">mode</span><span class="tok-o">(</span><span class="tok-s">&quot;overwrite&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">format</span><span class="tok-o">(</span><span class="tok-s">&quot;com.mongodb.spark.sql&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">save</span><span class="tok-o">()</span>

    <span class="tok-c1">//关闭Spark</span>
    <span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">close</span><span class="tok-o">()</span>
  <span class="tok-o">}</span>
<span class="tok-o">}</span></code></pre>
</div>
</div>
<div class="paragraph">
<p>然后通过商品特征向量进而求出相似度矩阵，就可以为实时推荐提供基础，得到用户推荐列表了。可以看出，基于内容和基于隐语义模型，目的都是为了提取出物品的特征向量，从而可以计算出相似度矩阵。而我们的实时推荐系统算法正是基于相似度来定义的。</p>
</div>
<div class="paragraph">
<p>当然，别忘记在`pom.xml`中添加依赖：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="xml"><span></span>    <span class="tok-nt">&lt;dependencies&gt;</span>

        <span class="tok-c">&lt;!-- 引入Spark相关的Jar包 --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>spark-core_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>2.1.1<span class="tok-nt">&lt;/version&gt;</span>
            <span class="tok-c">&lt;!-- provider如果存在，那么运行时该Jar包不存在，也不会打包到最终的发布版本中，只是编译器有效 --&gt;</span>
            <span class="tok-c">&lt;!--&lt;scope&gt;provided&lt;/scope&gt;--&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>spark-sql_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>2.1.1<span class="tok-nt">&lt;/version&gt;</span>
            <span class="tok-c">&lt;!-- provider如果存在，那么运行时该Jar包不存在，也不会打包到最终的发布版本中，只是编译器有效 --&gt;</span>
            <span class="tok-c">&lt;!--&lt;scope&gt;provided&lt;/scope&gt;--&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>spark-streaming_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>2.1.1<span class="tok-nt">&lt;/version&gt;</span>
            <span class="tok-c">&lt;!-- provider如果存在，那么运行时该Jar包不存在，也不会打包到最终的发布版本中，只是编译器有效 --&gt;</span>
            <span class="tok-c">&lt;!--&lt;scope&gt;provided&lt;/scope&gt;--&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>spark-mllib_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>2.1.1<span class="tok-nt">&lt;/version&gt;</span>
            <span class="tok-c">&lt;!-- provider如果存在，那么运行时该Jar包不存在，也不会打包到最终的发布版本中，只是编译器有效 --&gt;</span>
            <span class="tok-c">&lt;!--&lt;scope&gt;provided&lt;/scope&gt;--&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>spark-graphx_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>2.1.1<span class="tok-nt">&lt;/version&gt;</span>
            <span class="tok-c">&lt;!-- provider如果存在，那么运行时该Jar包不存在，也不会打包到最终的发布版本中，只是编译器有效 --&gt;</span>
            <span class="tok-c">&lt;!--&lt;scope&gt;provided&lt;/scope&gt;--&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>

        <span class="tok-c">&lt;!-- 加入MongoDB的驱动 --&gt;</span>
        <span class="tok-c">&lt;!-- 用于代码方式连接MongoDB --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.mongodb<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>casbah-core_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${casbah.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-c">&lt;!-- 用于Spark和MongoDB的对接 --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.mongodb.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>mongo-spark-connector_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${mongodb-spark.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>

        <span class="tok-c">&lt;!-- 引入Scala --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.scala-lang<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>scala-library<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${scala.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>

        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.scalanlp<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>jblas<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${jblas.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>

    <span class="tok-nt">&lt;/dependencies&gt;</span></code></pre>
</div>
</div>
</div>
</div>
<div class="sect2">
<h3 id="_基于物品的协同过滤">5.12. 基于物品的协同过滤</h3>
<div class="paragraph">
<p>基于同现相似度的计算公式来计算不同物品间的相似度。</p>
</div>
<div class="stemblock">
<div class="content">
\[w_{ij} = \frac {\vert N_i \cap N_j \vert} {\sqrt {\vert N_i \vert \vert N_j \vert}}\]
</div>
</div>
<div class="paragraph">
<p>其中$N_i$为购买商品\(i\)的用户列表，$N_j$为购买商品$j$的用户列表。</p>
</div>
<div class="paragraph">
<p>核心代码如下：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="scala"><span></span><span class="tok-k">package</span> <span class="tok-nn">com.atguigu.itemcf</span>

<span class="tok-k">import</span> <span class="tok-nn">org.apache.spark.SparkConf</span>
<span class="tok-k">import</span> <span class="tok-nn">org.apache.spark.sql.SparkSession</span>

<span class="tok-c1">//  物品信息</span>
<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">Product</span><span class="tok-o">(</span><span class="tok-n">productId</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">name</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">,</span> <span class="tok-n">categories</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">,</span> <span class="tok-n">imageUrl</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">,</span> <span class="tok-n">tags</span><span class="tok-k">:</span> <span class="tok-kt">String</span><span class="tok-o">)</span>

<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">MongoConfig</span><span class="tok-o">(</span><span class="tok-n">uri</span><span class="tok-k">:</span><span class="tok-kt">String</span><span class="tok-o">,</span> <span class="tok-n">db</span><span class="tok-k">:</span><span class="tok-kt">String</span><span class="tok-o">)</span>

<span class="tok-c1">//  用户-物品-评分</span>
<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">Rating</span><span class="tok-o">(</span><span class="tok-n">userId</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">productId</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">score</span><span class="tok-k">:</span> <span class="tok-kt">Double</span><span class="tok-o">,</span> <span class="tok-n">timestamp</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">)</span>

<span class="tok-c1">//  用户信息</span>
<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">User</span><span class="tok-o">(</span><span class="tok-n">userId</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">)</span>

<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">Recommendation</span><span class="tok-o">(</span><span class="tok-n">rid</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">r</span><span class="tok-k">:</span> <span class="tok-kt">Double</span><span class="tok-o">)</span>

<span class="tok-k">case</span> <span class="tok-k">class</span> <span class="tok-nc">ProductRecs</span><span class="tok-o">(</span><span class="tok-n">productId</span><span class="tok-k">:</span> <span class="tok-kt">Int</span><span class="tok-o">,</span> <span class="tok-n">recs</span><span class="tok-k">:</span><span class="tok-kt">Seq</span><span class="tok-o">[</span><span class="tok-kt">Recommendation</span><span class="tok-o">])</span>

<span class="tok-k">object</span> <span class="tok-nc">ItemCFRecommender</span> <span class="tok-o">{</span>
  <span class="tok-c1">// 同现相似度计算公式</span>
  <span class="tok-c1">// 比如：对A评分的人数100，对B评分的人数100，交集人数20</span>
  <span class="tok-c1">// 同现相似度：20 / 100 = 0.2</span>
  <span class="tok-k">def</span> <span class="tok-n">cooccurrence</span><span class="tok-o">(</span><span class="tok-n">numOfRatersForAAndB</span><span class="tok-k">:</span> <span class="tok-kt">Long</span><span class="tok-o">,</span> <span class="tok-n">numOfRatersForA</span><span class="tok-k">:</span> <span class="tok-kt">Long</span><span class="tok-o">,</span> <span class="tok-n">numOfRatersForB</span><span class="tok-k">:</span> <span class="tok-kt">Long</span><span class="tok-o">)</span><span class="tok-k">:</span> <span class="tok-kt">Double</span> <span class="tok-o">=</span> <span class="tok-o">{</span>
    <span class="tok-n">numOfRatersForAAndB</span> <span class="tok-o">/</span> <span class="tok-n">math</span><span class="tok-o">.</span><span class="tok-n">sqrt</span><span class="tok-o">(</span><span class="tok-n">numOfRatersForA</span> <span class="tok-o">*</span> <span class="tok-n">numOfRatersForB</span><span class="tok-o">)</span>
  <span class="tok-o">}</span>

  <span class="tok-k">val</span> <span class="tok-nc">MONGODB_PRODUCT_COLLECTION</span> <span class="tok-k">=</span> <span class="tok-s">&quot;Products&quot;</span>
  <span class="tok-k">val</span> <span class="tok-nc">MONGODB_RATING_COLLECTION</span> <span class="tok-k">=</span> <span class="tok-s">&quot;Rating&quot;</span>
  <span class="tok-k">val</span> <span class="tok-nc">PRODUCT_RECS</span> <span class="tok-k">=</span> <span class="tok-s">&quot;ItemCFProductRecs&quot;</span>


  <span class="tok-k">def</span> <span class="tok-n">main</span><span class="tok-o">(</span><span class="tok-n">args</span><span class="tok-k">:</span> <span class="tok-kt">Array</span><span class="tok-o">[</span><span class="tok-kt">String</span><span class="tok-o">])</span><span class="tok-k">:</span> <span class="tok-kt">Unit</span> <span class="tok-o">=</span> <span class="tok-o">{</span>
    <span class="tok-k">val</span> <span class="tok-n">config</span> <span class="tok-k">=</span> <span class="tok-nc">Map</span><span class="tok-o">(</span>
      <span class="tok-s">&quot;spark.cores&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-s">&quot;local[*]&quot;</span><span class="tok-o">,</span>
      <span class="tok-s">&quot;mongo.uri&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-s">&quot;mongodb://localhost:27017/recommender&quot;</span><span class="tok-o">,</span>
      <span class="tok-s">&quot;mongo.db&quot;</span> <span class="tok-o">-&gt;</span> <span class="tok-s">&quot;reommender&quot;</span>
    <span class="tok-o">)</span>

    <span class="tok-c1">//创建一个SparkConf配置</span>
    <span class="tok-k">val</span> <span class="tok-n">sparkConf</span> <span class="tok-k">=</span> <span class="tok-k">new</span> <span class="tok-nc">SparkConf</span><span class="tok-o">().</span><span class="tok-n">setAppName</span><span class="tok-o">(</span><span class="tok-s">&quot;ItemCFRecommender&quot;</span><span class="tok-o">).</span><span class="tok-n">setMaster</span><span class="tok-o">(</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-s">&quot;spark.cores&quot;</span><span class="tok-o">)).</span><span class="tok-n">set</span><span class="tok-o">(</span><span class="tok-s">&quot;spark.executor.memory&quot;</span><span class="tok-o">,</span><span class="tok-s">&quot;6G&quot;</span><span class="tok-o">).</span><span class="tok-n">set</span><span class="tok-o">(</span><span class="tok-s">&quot;spark.driver.memory&quot;</span><span class="tok-o">,</span><span class="tok-s">&quot;2G&quot;</span><span class="tok-o">)</span>

    <span class="tok-c1">//基于SparkConf创建一个SparkSession</span>
    <span class="tok-k">val</span> <span class="tok-n">spark</span> <span class="tok-k">=</span> <span class="tok-nc">SparkSession</span><span class="tok-o">.</span><span class="tok-n">builder</span><span class="tok-o">().</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-n">sparkConf</span><span class="tok-o">).</span><span class="tok-n">getOrCreate</span><span class="tok-o">()</span>

    <span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">sparkContext</span><span class="tok-o">.</span><span class="tok-n">setLogLevel</span><span class="tok-o">(</span><span class="tok-s">&quot;ERROR&quot;</span><span class="tok-o">)</span>

    <span class="tok-c1">//创建一个MongoDBConfig</span>
    <span class="tok-k">val</span> <span class="tok-n">mongoConfig</span> <span class="tok-k">=</span> <span class="tok-nc">MongoConfig</span><span class="tok-o">(</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-s">&quot;mongo.uri&quot;</span><span class="tok-o">),</span><span class="tok-n">config</span><span class="tok-o">(</span><span class="tok-s">&quot;mongo.db&quot;</span><span class="tok-o">))</span>

    <span class="tok-k">import</span> <span class="tok-nn">spark.implicits._</span>

    <span class="tok-c1">// 读取mongoDB中的业务数据</span>
    <span class="tok-k">val</span> <span class="tok-n">ratingDF</span> <span class="tok-k">=</span> <span class="tok-n">spark</span>
      <span class="tok-o">.</span><span class="tok-n">read</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;uri&quot;</span><span class="tok-o">,</span> <span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">uri</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;collection&quot;</span><span class="tok-o">,</span> <span class="tok-nc">MONGODB_RATING_COLLECTION</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">format</span><span class="tok-o">(</span><span class="tok-s">&quot;com.mongodb.spark.sql&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">load</span><span class="tok-o">()</span>
      <span class="tok-o">.</span><span class="tok-n">as</span><span class="tok-o">[</span><span class="tok-kt">Rating</span><span class="tok-o">]</span>
      <span class="tok-o">.</span><span class="tok-n">rdd</span>
      <span class="tok-o">.</span><span class="tok-n">map</span> <span class="tok-o">{</span>
        <span class="tok-n">rating</span> <span class="tok-k">=&gt;</span> <span class="tok-o">{</span>
          <span class="tok-o">(</span><span class="tok-n">rating</span><span class="tok-o">.</span><span class="tok-n">userId</span><span class="tok-o">,</span> <span class="tok-n">rating</span><span class="tok-o">.</span><span class="tok-n">productId</span><span class="tok-o">,</span> <span class="tok-n">rating</span><span class="tok-o">.</span><span class="tok-n">score</span><span class="tok-o">)</span>
        <span class="tok-o">}</span>
      <span class="tok-o">}</span>
      <span class="tok-o">.</span><span class="tok-n">cache</span><span class="tok-o">()</span>
      <span class="tok-o">.</span><span class="tok-n">toDF</span><span class="tok-o">(</span><span class="tok-s">&quot;userId&quot;</span><span class="tok-o">,</span> <span class="tok-s">&quot;productId&quot;</span><span class="tok-o">,</span> <span class="tok-s">&quot;rating&quot;</span><span class="tok-o">)</span>

    <span class="tok-k">val</span> <span class="tok-n">numRatersPerProduct</span> <span class="tok-k">=</span> <span class="tok-n">ratingDF</span><span class="tok-o">.</span><span class="tok-n">groupBy</span><span class="tok-o">(</span><span class="tok-s">&quot;productId&quot;</span><span class="tok-o">).</span><span class="tok-n">count</span><span class="tok-o">().</span><span class="tok-n">alias</span><span class="tok-o">(</span><span class="tok-s">&quot;nor&quot;</span><span class="tok-o">)</span>

    <span class="tok-c1">// 在原记录基础上加上product的打分者的数量</span>
    <span class="tok-k">val</span> <span class="tok-n">ratingsWithSize</span> <span class="tok-k">=</span> <span class="tok-n">ratingDF</span><span class="tok-o">.</span><span class="tok-n">join</span><span class="tok-o">(</span><span class="tok-n">numRatersPerProduct</span><span class="tok-o">,</span> <span class="tok-s">&quot;productId&quot;</span><span class="tok-o">)</span>

    <span class="tok-c1">// 执行内联操作</span>
    <span class="tok-k">val</span> <span class="tok-n">joinedDF</span> <span class="tok-k">=</span> <span class="tok-n">ratingsWithSize</span><span class="tok-o">.</span><span class="tok-n">join</span><span class="tok-o">(</span><span class="tok-n">ratingsWithSize</span><span class="tok-o">,</span> <span class="tok-s">&quot;userId&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">toDF</span><span class="tok-o">(</span><span class="tok-s">&quot;userId&quot;</span><span class="tok-o">,</span> <span class="tok-s">&quot;product1&quot;</span><span class="tok-o">,</span> <span class="tok-s">&quot;rating1&quot;</span><span class="tok-o">,</span> <span class="tok-s">&quot;nor1&quot;</span><span class="tok-o">,</span> <span class="tok-s">&quot;product2&quot;</span><span class="tok-o">,</span> <span class="tok-s">&quot;rating2&quot;</span><span class="tok-o">,</span> <span class="tok-s">&quot;nor2&quot;</span><span class="tok-o">)</span>

    <span class="tok-n">joinedDF</span>
      <span class="tok-o">.</span><span class="tok-n">selectExpr</span><span class="tok-o">(</span><span class="tok-s">&quot;userId&quot;</span><span class="tok-o">,</span> <span class="tok-s">&quot;product1&quot;</span><span class="tok-o">,</span> <span class="tok-s">&quot;nor1&quot;</span><span class="tok-o">,</span> <span class="tok-s">&quot;product2&quot;</span><span class="tok-o">,</span> <span class="tok-s">&quot;nor2&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">createOrReplaceTempView</span><span class="tok-o">(</span><span class="tok-s">&quot;joined&quot;</span><span class="tok-o">)</span>

    <span class="tok-c1">//  计算必要的中间数据，注意此处有WHERE限定，只计算了一半的数据量</span>
    <span class="tok-k">val</span> <span class="tok-n">sparseMatrix</span> <span class="tok-k">=</span> <span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">sql</span><span class="tok-o">(</span>
      <span class="tok-s">&quot;&quot;&quot;</span>
<span class="tok-s">        |SELECT product1</span>
<span class="tok-s">        |, product2</span>
<span class="tok-s">        |, count(userId) as size</span>
<span class="tok-s">        |, first(nor1) as nor1</span>
<span class="tok-s">        |, first(nor2) as nor2</span>
<span class="tok-s">        |FROM joined</span>
<span class="tok-s">        |GROUP BY product1, product2</span>
<span class="tok-s">      &quot;&quot;&quot;</span><span class="tok-o">.</span><span class="tok-n">stripMargin</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">cache</span><span class="tok-o">()</span>

    <span class="tok-c1">//  计算物品相似度</span>
    <span class="tok-k">var</span> <span class="tok-n">sim</span> <span class="tok-k">=</span> <span class="tok-n">sparseMatrix</span><span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">row</span> <span class="tok-k">=&gt;</span> <span class="tok-o">{</span>
      <span class="tok-k">val</span> <span class="tok-n">size</span> <span class="tok-k">=</span> <span class="tok-n">row</span><span class="tok-o">.</span><span class="tok-n">getAs</span><span class="tok-o">[</span><span class="tok-kt">Long</span><span class="tok-o">](</span><span class="tok-mi">2</span><span class="tok-o">)</span>
      <span class="tok-k">val</span> <span class="tok-n">numRaters1</span> <span class="tok-k">=</span> <span class="tok-n">row</span><span class="tok-o">.</span><span class="tok-n">getAs</span><span class="tok-o">[</span><span class="tok-kt">Long</span><span class="tok-o">](</span><span class="tok-mi">3</span><span class="tok-o">)</span>
      <span class="tok-k">val</span> <span class="tok-n">numRaters2</span> <span class="tok-k">=</span> <span class="tok-n">row</span><span class="tok-o">.</span><span class="tok-n">getAs</span><span class="tok-o">[</span><span class="tok-kt">Long</span><span class="tok-o">](</span><span class="tok-mi">4</span><span class="tok-o">)</span>

      <span class="tok-k">val</span> <span class="tok-n">cooc</span> <span class="tok-k">=</span> <span class="tok-n">cooccurrence</span><span class="tok-o">(</span><span class="tok-n">size</span><span class="tok-o">,</span> <span class="tok-n">numRaters1</span><span class="tok-o">,</span> <span class="tok-n">numRaters2</span><span class="tok-o">)</span>
      <span class="tok-o">(</span><span class="tok-n">row</span><span class="tok-o">.</span><span class="tok-n">getInt</span><span class="tok-o">(</span><span class="tok-mi">0</span><span class="tok-o">),</span> <span class="tok-n">row</span><span class="tok-o">.</span><span class="tok-n">getInt</span><span class="tok-o">(</span><span class="tok-mi">1</span><span class="tok-o">),</span> <span class="tok-n">cooc</span><span class="tok-o">)</span>
    <span class="tok-o">}).</span><span class="tok-n">toDF</span><span class="tok-o">(</span><span class="tok-s">&quot;productId_01&quot;</span><span class="tok-o">,</span> <span class="tok-s">&quot;productId_02&quot;</span><span class="tok-o">,</span> <span class="tok-s">&quot;cooc&quot;</span><span class="tok-o">)</span>

    <span class="tok-k">val</span> <span class="tok-n">simDF</span> <span class="tok-k">=</span> <span class="tok-n">sim</span>
      <span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">{</span>
        <span class="tok-k">case</span> <span class="tok-n">row</span> <span class="tok-k">=&gt;</span> <span class="tok-o">(</span>
          <span class="tok-n">row</span><span class="tok-o">.</span><span class="tok-n">getAs</span><span class="tok-o">[</span><span class="tok-kt">Int</span><span class="tok-o">](</span><span class="tok-s">&quot;productId_01&quot;</span><span class="tok-o">),</span>
          <span class="tok-n">row</span><span class="tok-o">.</span><span class="tok-n">getAs</span><span class="tok-o">[</span><span class="tok-kt">Int</span><span class="tok-o">](</span><span class="tok-s">&quot;productId_02&quot;</span><span class="tok-o">),</span>
          <span class="tok-n">row</span><span class="tok-o">.</span><span class="tok-n">getAs</span><span class="tok-o">[</span><span class="tok-kt">Double</span><span class="tok-o">](</span><span class="tok-s">&quot;cooc&quot;</span><span class="tok-o">)</span>
        <span class="tok-o">)</span>
      <span class="tok-o">}</span>
      <span class="tok-o">.</span><span class="tok-n">rdd</span>
      <span class="tok-o">.</span><span class="tok-n">map</span><span class="tok-o">(</span>
        <span class="tok-n">x</span> <span class="tok-k">=&gt;</span> <span class="tok-o">(</span><span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">,</span> <span class="tok-o">(</span><span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">,</span> <span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_3</span><span class="tok-o">))</span>
      <span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">groupByKey</span><span class="tok-o">()</span>
      <span class="tok-o">.</span><span class="tok-n">map</span> <span class="tok-o">{</span>
        <span class="tok-k">case</span> <span class="tok-o">(</span><span class="tok-n">productId</span><span class="tok-o">,</span> <span class="tok-n">items</span><span class="tok-o">)</span> <span class="tok-k">=&gt;</span> <span class="tok-nc">ProductRecs</span><span class="tok-o">(</span><span class="tok-n">productId</span><span class="tok-o">,</span> <span class="tok-n">items</span><span class="tok-o">.</span><span class="tok-n">toList</span><span class="tok-o">.</span><span class="tok-n">filter</span><span class="tok-o">(</span><span class="tok-n">x</span> <span class="tok-k">=&gt;</span> <span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_1</span> <span class="tok-o">!=</span> <span class="tok-n">productId</span><span class="tok-o">).</span><span class="tok-n">sortWith</span><span class="tok-o">(</span><span class="tok-k">_</span><span class="tok-o">.</span><span class="tok-n">_2</span> <span class="tok-o">&gt;</span> <span class="tok-k">_</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">).</span><span class="tok-n">map</span><span class="tok-o">(</span><span class="tok-n">x</span> <span class="tok-k">=&gt;</span> <span class="tok-nc">Recommendation</span><span class="tok-o">(</span><span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_1</span><span class="tok-o">,</span><span class="tok-n">x</span><span class="tok-o">.</span><span class="tok-n">_2</span><span class="tok-o">)).</span><span class="tok-n">take</span><span class="tok-o">(</span><span class="tok-mi">5</span><span class="tok-o">))</span>
      <span class="tok-o">}</span>
      <span class="tok-o">.</span><span class="tok-n">toDF</span><span class="tok-o">()</span>

    <span class="tok-n">simDF</span>
      <span class="tok-o">.</span><span class="tok-n">write</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;uri&quot;</span><span class="tok-o">,</span> <span class="tok-n">mongoConfig</span><span class="tok-o">.</span><span class="tok-n">uri</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">option</span><span class="tok-o">(</span><span class="tok-s">&quot;collection&quot;</span><span class="tok-o">,</span> <span class="tok-nc">PRODUCT_RECS</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">mode</span><span class="tok-o">(</span><span class="tok-s">&quot;overwrite&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">format</span><span class="tok-o">(</span><span class="tok-s">&quot;com.mongodb.spark.sql&quot;</span><span class="tok-o">)</span>
      <span class="tok-o">.</span><span class="tok-n">save</span><span class="tok-o">()</span>

    <span class="tok-c1">//关闭Spark</span>
    <span class="tok-n">spark</span><span class="tok-o">.</span><span class="tok-n">close</span><span class="tok-o">()</span>

  <span class="tok-o">}</span>

<span class="tok-o">}</span></code></pre>
</div>
</div>
<div class="paragraph">
<p>别忘了添加依赖：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="xml"><span></span>    <span class="tok-nt">&lt;dependencies&gt;</span>

        <span class="tok-c">&lt;!-- 引入Spark相关的Jar包 --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>spark-core_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>2.1.1<span class="tok-nt">&lt;/version&gt;</span>
            <span class="tok-c">&lt;!-- provider如果存在，那么运行时该Jar包不存在，也不会打包到最终的发布版本中，只是编译器有效 --&gt;</span>
            <span class="tok-c">&lt;!--&lt;scope&gt;provided&lt;/scope&gt;--&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>spark-sql_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>2.1.1<span class="tok-nt">&lt;/version&gt;</span>
            <span class="tok-c">&lt;!-- provider如果存在，那么运行时该Jar包不存在，也不会打包到最终的发布版本中，只是编译器有效 --&gt;</span>
            <span class="tok-c">&lt;!--&lt;scope&gt;provided&lt;/scope&gt;--&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>spark-streaming_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>2.1.1<span class="tok-nt">&lt;/version&gt;</span>
            <span class="tok-c">&lt;!-- provider如果存在，那么运行时该Jar包不存在，也不会打包到最终的发布版本中，只是编译器有效 --&gt;</span>
            <span class="tok-c">&lt;!--&lt;scope&gt;provided&lt;/scope&gt;--&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>spark-mllib_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>2.1.1<span class="tok-nt">&lt;/version&gt;</span>
            <span class="tok-c">&lt;!-- provider如果存在，那么运行时该Jar包不存在，也不会打包到最终的发布版本中，只是编译器有效 --&gt;</span>
            <span class="tok-c">&lt;!--&lt;scope&gt;provided&lt;/scope&gt;--&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.apache.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>spark-graphx_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>2.1.1<span class="tok-nt">&lt;/version&gt;</span>
            <span class="tok-c">&lt;!-- provider如果存在，那么运行时该Jar包不存在，也不会打包到最终的发布版本中，只是编译器有效 --&gt;</span>
            <span class="tok-c">&lt;!--&lt;scope&gt;provided&lt;/scope&gt;--&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>

        <span class="tok-c">&lt;!-- 加入MongoDB的驱动 --&gt;</span>
        <span class="tok-c">&lt;!-- 用于代码方式连接MongoDB --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.mongodb<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>casbah-core_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${casbah.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>
        <span class="tok-c">&lt;!-- 用于Spark和MongoDB的对接 --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.mongodb.spark<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>mongo-spark-connector_2.11<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${mongodb-spark.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>

        <span class="tok-c">&lt;!-- 引入Scala --&gt;</span>
        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.scala-lang<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>scala-library<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${scala.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>

        <span class="tok-nt">&lt;dependency&gt;</span>
            <span class="tok-nt">&lt;groupId&gt;</span>org.scalanlp<span class="tok-nt">&lt;/groupId&gt;</span>
            <span class="tok-nt">&lt;artifactId&gt;</span>jblas<span class="tok-nt">&lt;/artifactId&gt;</span>
            <span class="tok-nt">&lt;version&gt;</span>${jblas.version}<span class="tok-nt">&lt;/version&gt;</span>
        <span class="tok-nt">&lt;/dependency&gt;</span>

    <span class="tok-nt">&lt;/dependencies&gt;</span></code></pre>
</div>
</div>
</div>
<div class="sect2">
<h3 id="_附记">5.13. 附记</h3>
<div class="olist arabic">
<ol class="arabic">
<li>
<p>DataLoader用来将csv文件中的数据导入MongoDB.</p>
</li>
<li>
<p>StatisticRecommender统计一些热门商品和评分最多的商品，用来解决冷启动问题。显示在首页的`热门推荐`和`评分最多`标签下。</p>
</li>
<li>
<p>OfflineRecommender: ALS矩阵分解算法。</p>
<div class="ulist">
<ul>
<li>
<p>UserRecs: 矩阵分解再相乘得到的预测评分矩阵降序排列，可以对用户进行推荐。显示在首页的`离线推荐`标签下。注意：注册完用户以后，需要完成评分操作，再运行OfflineRecommender.scala.</p>
</li>
<li>
<p>ProductRecs: 使用物品隐特征矩阵来两两计算余弦相似度。</p>
</li>
</ul>
</div>
</li>
<li>
<p>ItemCFRecommender: 使用同现相似度来计算物品之间的相似性。显示在商品详情页的`基于物品的协同过滤`标签下。</p>
</li>
<li>
<p>ContentBasedRecommender: 基于商品的标签的TFIDF权重向量，来计算余弦相似度。显示在商品详情页的`基于内容的推荐`标签下。</p>
</li>
<li>
<p>StreamingRecommender: 使用的物品之间相似度是使用的物品隐特征矩阵算出来的余弦相似度。然后和Redis里面保存的最近评分进行融合。显示在`实时推荐`下。</p>
</li>
<li>
<p>KafkaStream模块用来从topic=lo消费数据，并发送到topic=recommender中。</p>
</li>
</ol>
</div>
</div>
<div class="sect2">
<h3 id="_算法使用场景">5.14. 算法使用场景：</h3>
<div class="ulist">
<ul>
<li>
<p>ItemCF: 适用于电商电影类网站。</p>
</li>
<li>
<p>ContentBased: 适用于新闻类网站。</p>
</li>
<li>
<p>ALS: 都可以，矩阵分解主要用来做降维。</p>
</li>
</ul>
</div>
</div>
</div>
</div>
<div class="sect1">
<h2 id="_用python从头实现神经网络">6. 用Python从头实现神经网络</h2>
<div class="sectionbody">
<div class="sect2">
<h3 id="_砖块神经元">6.1. 砖块：神经元</h3>
<div class="paragraph">
<p>首先让我们看看神经网络的基本单位，神经元。神经元接受输入，对其做一些数据操作，然后产生输出。例如，这是一个2-输入神经元：</p>
</div>
<div class="imageblock">
<div class="content">
<img src="images/dnn1.svg" alt="dnn1">
</div>
</div>
<div class="paragraph">
<p>这里发生了三个事情。首先，每个输入都跟一个权重相乘（红色）：</p>
</div>
<div class="stemblock">
<div class="content">
\[x_1←x_1 \times w_1 \\
x_2←x_2 \times w_2\]
</div>
</div>
<div class="paragraph">
<p>然后，加权后的输入求和，加上一个偏差b（绿色）：</p>
</div>
<div class="stemblock">
<div class="content">
\[(x_1 \times w_1 )+(x_2 \times w_2 )+b\]
</div>
</div>
<div class="paragraph">
<p>最后，这个结果传递给一个激活函数f：</p>
</div>
<div class="stemblock">
<div class="content">
\[y=f(x_1 \times w_1+x_2 \times w_2+b)\]
</div>
</div>
<div class="paragraph">
<p>激活函数的用途是将一个无边界的输入，转变成一个可预测的形式。常用的激活函数就就是S型函数：</p>
</div>
<div class="imageblock">
<div class="content">
<img src="images/dnn2.png" alt="dnn2">
</div>
</div>
<div class="paragraph">
<p>S型函数的值域是(0, 1)。简单来说，就是把(−∞, +∞)压缩到(0, 1) ，很大的负数约等于0，很大的正数约等于1。</p>
</div>
</div>
<div class="sect2">
<h3 id="_一个简单的例子">6.2. 一个简单的例子</h3>
<div class="paragraph">
<p>假设我们有一个神经元，激活函数就是S型函数，其参数如下：</p>
</div>
<div class="stemblock">
<div class="content">
\[w=[0, 1] \\
b=4\]
</div>
</div>
<div class="paragraph">
<p>w=[0, 1] 就是以向量的形式表示w_1=0, w_2=1。现在，我们给这个神经元一个输入x=[2, 3]。我们用点积来表示：</p>
</div>
<div class="stemblock">
<div class="content">
\[\begin{align}
  (w ⋅x)+b &amp;=&amp; (w_1∗x_1+w_2∗x_2 )+b \\
           &amp;=&amp; 0∗2+1∗3+4 \\
           &amp;=&amp; 7
\end{align}\]
</div>
</div>
<div class="stemblock">
<div class="content">
\[y=f(w⋅x+b)=f(7)=0.999\]
</div>
</div>
<div class="paragraph">
<p>当输入是[2, 3]时，这个神经元的输出是0.999。给定输入，得到输出的过程被称为前馈（feedforward）。</p>
</div>
</div>
<div class="sect2">
<h3 id="_编码一个神经元">6.3. 编码一个神经元</h3>
<div class="paragraph">
<p>让我们来实现一个神经元！用Python的NumPy库来完成其中的数学计算：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="py"><span></span><span class="tok-kn">import</span> <span class="tok-nn">numpy</span> <span class="tok-kn">as</span> <span class="tok-nn">np</span>

<span class="tok-k">def</span> <span class="tok-nf">sigmoid</span><span class="tok-p">(</span><span class="tok-n">x</span><span class="tok-p">):</span>
  <span class="tok-c1"># Our activation function: f(x) = 1 / (1 + e^(-x))</span>
  <span class="tok-k">return</span> <span class="tok-mi">1</span> <span class="tok-o">/</span> <span class="tok-p">(</span><span class="tok-mi">1</span> <span class="tok-o">+</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">exp</span><span class="tok-p">(</span><span class="tok-o">-</span><span class="tok-n">x</span><span class="tok-p">))</span>

<span class="tok-k">class</span> <span class="tok-nc">Neuron</span><span class="tok-p">:</span>
  <span class="tok-k">def</span> <span class="tok-fm">__init__</span><span class="tok-p">(</span><span class="tok-bp">self</span><span class="tok-p">,</span> <span class="tok-n">weights</span><span class="tok-p">,</span> <span class="tok-n">bias</span><span class="tok-p">):</span>
    <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">weights</span> <span class="tok-o">=</span> <span class="tok-n">weights</span>
    <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">bias</span> <span class="tok-o">=</span> <span class="tok-n">bias</span>

  <span class="tok-k">def</span> <span class="tok-nf">feedforward</span><span class="tok-p">(</span><span class="tok-bp">self</span><span class="tok-p">,</span> <span class="tok-n">inputs</span><span class="tok-p">):</span>
    <span class="tok-c1"># Weight inputs, add bias, then use the activation function</span>
    <span class="tok-n">total</span> <span class="tok-o">=</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">dot</span><span class="tok-p">(</span><span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">weights</span><span class="tok-p">,</span> <span class="tok-n">inputs</span><span class="tok-p">)</span> <span class="tok-o">+</span> <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">bias</span>
    <span class="tok-k">return</span> <span class="tok-n">sigmoid</span><span class="tok-p">(</span><span class="tok-n">total</span><span class="tok-p">)</span>

<span class="tok-n">weights</span> <span class="tok-o">=</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">array</span><span class="tok-p">([</span><span class="tok-mi">0</span><span class="tok-p">,</span> <span class="tok-mi">1</span><span class="tok-p">])</span> <span class="tok-c1"># w1 = 0, w2 = 1</span>
<span class="tok-n">bias</span> <span class="tok-o">=</span> <span class="tok-mi">4</span>                   <span class="tok-c1"># b = 4</span>
<span class="tok-n">n</span> <span class="tok-o">=</span> <span class="tok-n">Neuron</span><span class="tok-p">(</span><span class="tok-n">weights</span><span class="tok-p">,</span> <span class="tok-n">bias</span><span class="tok-p">)</span>

<span class="tok-n">x</span> <span class="tok-o">=</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">array</span><span class="tok-p">([</span><span class="tok-mi">2</span><span class="tok-p">,</span> <span class="tok-mi">3</span><span class="tok-p">])</span>       <span class="tok-c1"># x1 = 2, x2 = 3</span>
<span class="tok-k">print</span><span class="tok-p">(</span><span class="tok-n">n</span><span class="tok-o">.</span><span class="tok-n">feedforward</span><span class="tok-p">(</span><span class="tok-n">x</span><span class="tok-p">))</span>    <span class="tok-c1"># 0.9990889488055994</span></code></pre>
</div>
</div>
<div class="paragraph">
<p>还记得这个数字吗？就是我们前面算出来的例子中的0.999。</p>
</div>
</div>
<div class="sect2">
<h3 id="_把神经元组装成网络">6.4. 把神经元组装成网络</h3>
<div class="paragraph">
<p>所谓的神经网络就是一堆神经元。这就是一个简单的神经网络：</p>
</div>
<div class="imageblock">
<div class="content">
<img src="images/dnn3.svg" alt="dnn3">
</div>
</div>
<div class="paragraph">
<p>这个网络有两个输入，一个有两个神经元（\(h_1\)和\(h_2\)）的隐藏层，以及一个有一个神经元（\(o_1\)）的输出层。要注意，\(o_1\)的输入就是\(h_1\)和\(h_2\)的输出，这样就组成了一个网络。</p>
</div>
<div class="admonitionblock note">
<table>
<tr>
<td class="icon">
<i class="fa icon-note" title="Note"></i>
</td>
<td class="content">
隐藏层就是输入层和输出层之间的层，隐藏层可以是多层的。
</td>
</tr>
</table>
</div>
</div>
<div class="sect2">
<h3 id="_例子前馈">6.5. 例子：前馈</h3>
<div class="paragraph">
<p>我们继续用前面图中的网络，假设每个神经元的权重都是\(w=[0,1\)]，截距项也相同\(b=0\)，激活函数也都是S型函数。分别用\(h_1, h_2, o_1\)表示相应的神经元的输出。</p>
</div>
<div class="paragraph">
<p>当输入\(x=[2,3\)]时，会得到什么结果？</p>
</div>
<div class="stemblock">
<div class="content">
\[\begin{align}  h1 = h2 &amp;= f(w \cdot x + b)\\  &amp;=f(0\times 2 + 1 \times 3 + 0)\\ &amp;=f(3)\\  &amp;=0.9526\\  o_1 &amp;= f(w \cdot [h_1, h_2] + b)\\  &amp;=f(0 \times h_1 + 1 \times h_2 + 0)\\ &amp;=f(0.9526)\\  &amp;=0.7216  \end{align}\]
</div>
</div>
<div class="paragraph">
<p>这个神经网络对输入[2,3]的输出是0.7216，很简单。</p>
</div>
<div class="paragraph">
<p>一个神经网络的层数以及每一层中的神经元数量都是任意的。基本逻辑都一样：输入在神经网络中向前传输，最终得到输出。接下来，我们会继续使用前面的这个网络。</p>
</div>
</div>
<div class="sect2">
<h3 id="_编码神经网络前馈">6.6. 编码神经网络：前馈</h3>
<div class="paragraph">
<p>接下来我们实现这个神经网络的前馈机制，还是这个图：</p>
</div>
<div class="imageblock">
<div class="content">
<img src="images/dnn3.svg" alt="dnn3">
</div>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="py"><span></span><span class="tok-kn">import</span> <span class="tok-nn">numpy</span> <span class="tok-kn">as</span> <span class="tok-nn">np</span>

<span class="tok-c1"># ... code from previous section here</span>

<span class="tok-k">class</span> <span class="tok-nc">OurNeuralNetwork</span><span class="tok-p">:</span>
  <span class="tok-sd">&#39;&#39;&#39;</span>
<span class="tok-sd">  A neural network with:</span>
<span class="tok-sd">    - 2 inputs</span>
<span class="tok-sd">    - a hidden layer with 2 neurons (h1, h2)</span>
<span class="tok-sd">    - an output layer with 1 neuron (o1)</span>
<span class="tok-sd">  Each neuron has the same weights and bias:</span>
<span class="tok-sd">    - w = [0, 1]</span>
<span class="tok-sd">    - b = 0</span>
<span class="tok-sd">  &#39;&#39;&#39;</span>
  <span class="tok-k">def</span> <span class="tok-fm">__init__</span><span class="tok-p">(</span><span class="tok-bp">self</span><span class="tok-p">):</span>
    <span class="tok-n">weights</span> <span class="tok-o">=</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">array</span><span class="tok-p">([</span><span class="tok-mi">0</span><span class="tok-p">,</span> <span class="tok-mi">1</span><span class="tok-p">])</span>
    <span class="tok-n">bias</span> <span class="tok-o">=</span> <span class="tok-mi">0</span>

    <span class="tok-c1"># The Neuron class here is from the previous section</span>
    <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">h1</span> <span class="tok-o">=</span> <span class="tok-n">Neuron</span><span class="tok-p">(</span><span class="tok-n">weights</span><span class="tok-p">,</span> <span class="tok-n">bias</span><span class="tok-p">)</span>
    <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">h2</span> <span class="tok-o">=</span> <span class="tok-n">Neuron</span><span class="tok-p">(</span><span class="tok-n">weights</span><span class="tok-p">,</span> <span class="tok-n">bias</span><span class="tok-p">)</span>
    <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">o1</span> <span class="tok-o">=</span> <span class="tok-n">Neuron</span><span class="tok-p">(</span><span class="tok-n">weights</span><span class="tok-p">,</span> <span class="tok-n">bias</span><span class="tok-p">)</span>

  <span class="tok-k">def</span> <span class="tok-nf">feedforward</span><span class="tok-p">(</span><span class="tok-bp">self</span><span class="tok-p">,</span> <span class="tok-n">x</span><span class="tok-p">):</span>
    <span class="tok-n">out_h1</span> <span class="tok-o">=</span> <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">h1</span><span class="tok-o">.</span><span class="tok-n">feedforward</span><span class="tok-p">(</span><span class="tok-n">x</span><span class="tok-p">)</span>
    <span class="tok-n">out_h2</span> <span class="tok-o">=</span> <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">h2</span><span class="tok-o">.</span><span class="tok-n">feedforward</span><span class="tok-p">(</span><span class="tok-n">x</span><span class="tok-p">)</span>

    <span class="tok-c1"># The inputs for o1 are the outputs from h1 and h2</span>
    <span class="tok-n">out_o1</span> <span class="tok-o">=</span> <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">o1</span><span class="tok-o">.</span><span class="tok-n">feedforward</span><span class="tok-p">(</span><span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">array</span><span class="tok-p">([</span><span class="tok-n">out_h1</span><span class="tok-p">,</span> <span class="tok-n">out_h2</span><span class="tok-p">]))</span>

    <span class="tok-k">return</span> <span class="tok-n">out_o1</span>

<span class="tok-n">network</span> <span class="tok-o">=</span> <span class="tok-n">OurNeuralNetwork</span><span class="tok-p">()</span>
<span class="tok-n">x</span> <span class="tok-o">=</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">array</span><span class="tok-p">([</span><span class="tok-mi">2</span><span class="tok-p">,</span> <span class="tok-mi">3</span><span class="tok-p">])</span>
<span class="tok-k">print</span><span class="tok-p">(</span><span class="tok-n">network</span><span class="tok-o">.</span><span class="tok-n">feedforward</span><span class="tok-p">(</span><span class="tok-n">x</span><span class="tok-p">))</span> <span class="tok-c1"># 0.7216325609518421</span></code></pre>
</div>
</div>
<div class="paragraph">
<p>结果正确，看上去没问题。</p>
</div>
</div>
<div class="sect2">
<h3 id="_训练神经网络第1部分">6.7. 训练神经网络，第1部分</h3>
<div class="paragraph">
<p>现在有这样的数据：</p>
</div>
<table class="tableblock frame-all grid-all stretch">
<colgroup>
<col style="width: 25%;">
<col style="width: 25%;">
<col style="width: 25%;">
<col style="width: 25%;">
</colgroup>
<thead>
<tr>
<th class="tableblock halign-left valign-top">Name</th>
<th class="tableblock halign-left valign-top">Weight</th>
<th class="tableblock halign-left valign-top">Height</th>
<th class="tableblock halign-left valign-top">Gender</th>
</tr>
</thead>
<tbody>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">Alice</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">133</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">65</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">F</p></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">Bob</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">160</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">72</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">M</p></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">Charlie</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">152</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">70</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">M</p></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">Diana</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">120</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">60</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">F</p></td>
</tr>
</tbody>
</table>
<div class="paragraph">
<p>接下来我们用这个数据来训练神经网络的权重和截距项，从而可以根据身高体重预测性别：</p>
</div>
<div class="imageblock">
<div class="content">
<img src="images/dnn4.svg" alt="dnn4">
</div>
</div>
<div class="paragraph">
<p>我们用0和1分别表示男性（M）和女性（F），并对数值做了转化：</p>
</div>
<table class="tableblock frame-all grid-all stretch">
<colgroup>
<col style="width: 25%;">
<col style="width: 25%;">
<col style="width: 25%;">
<col style="width: 25%;">
</colgroup>
<thead>
<tr>
<th class="tableblock halign-left valign-top">Name</th>
<th class="tableblock halign-left valign-top">Weight</th>
<th class="tableblock halign-left valign-top">Height</th>
<th class="tableblock halign-left valign-top">Gender</th>
</tr>
</thead>
<tbody>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">Alice</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">-2</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">-1</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">1</p></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">Bob</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">25</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">6</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">0</p></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">Charlie</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">17</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">4</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">0</p></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">Diana</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">-15</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">-6</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">1</p></td>
</tr>
</tbody>
</table>
<div class="admonitionblock note">
<table>
<tr>
<td class="icon">
<i class="fa icon-note" title="Note"></i>
</td>
<td class="content">
我这里是随意选取了135和66来标准化数据，通常会使用平均值。
</td>
</tr>
</table>
</div>
</div>
<div class="sect2">
<h3 id="_损失">6.8. 损失</h3>
<div class="paragraph">
<p>在训练网络之前，我们需要量化当前的网络是『好』还是『坏』，从而可以寻找更好的网络。这就是定义损失的目的。</p>
</div>
<div class="paragraph">
<p>我们在这里用平均方差（MSE）损失： \(MSE = \frac{1}{n}\sum_{i=1}^{n}(y_{true} - y_{pred})^2\) 让我们仔细看看：</p>
</div>
<div class="ulist">
<ul>
<li>
<p>n是样品数，这里等于4（Alice、Bob、Charlie和Diana）。</p>
</li>
<li>
<p>y表示要预测的变量，这里是性别。</p>
</li>
<li>
<p>\(y_{true}\) 是变量的真实值（『正确答案』）。例如，Alice的 \(y_{true}\) 就是1（男性）。</p>
</li>
<li>
<p>\(y_{pred}\) 是变量的预测值。这就是我们网络的输出。</p>
</li>
</ul>
</div>
<div class="paragraph">
<p>\((y_{true}-y_{pred})^2\)被称为方差（squared error）。我们的损失函数就是所有方差的平均值。预测效果于浩，损失就越少。</p>
</div>
<div class="paragraph">
<p>更好的预测 = 更少的损失！</p>
</div>
<div class="paragraph">
<p>训练网络 = 最小化它的损失。</p>
</div>
</div>
<div class="sect2">
<h3 id="_损失计算例子">6.9. 损失计算例子</h3>
<div class="paragraph">
<p>假设我们的网络总是输出0，换言之就是认为所有人都是男性。损失如何？</p>
</div>
<table class="tableblock frame-all grid-all stretch">
<colgroup>
<col style="width: 25%;">
<col style="width: 25%;">
<col style="width: 25%;">
<col style="width: 25%;">
</colgroup>
<thead>
<tr>
<th class="tableblock halign-left valign-top">Name</th>
<th class="tableblock halign-left valign-top">\(y_{true}\)</th>
<th class="tableblock halign-left valign-top">\(y_{pred}\)</th>
<th class="tableblock halign-left valign-top">\(y_{true}-y_{pred}\)</th>
</tr>
</thead>
<tbody>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">Alice</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">1</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">0</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">1</p></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">Bob</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">0</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">0</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">0</p></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">Charlie</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">0</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">0</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">0</p></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">Diana</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">1</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">0</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">1</p></td>
</tr>
</tbody>
</table>
<div class="stemblock">
<div class="content">
\[MSE = \frac{1}{4}(1+0+0+1)=0.5\]
</div>
</div>
</div>
<div class="sect2">
<h3 id="_代码mse损失">6.10. 代码：MSE损失</h3>
<div class="paragraph">
<p>下面是计算MSE损失的代码：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="py"><span></span><span class="tok-kn">import</span> <span class="tok-nn">numpy</span> <span class="tok-kn">as</span> <span class="tok-nn">np</span>

<span class="tok-k">def</span> <span class="tok-nf">mse_loss</span><span class="tok-p">(</span><span class="tok-n">y_true</span><span class="tok-p">,</span> <span class="tok-n">y_pred</span><span class="tok-p">):</span>
  <span class="tok-c1"># y_true and y_pred are numpy arrays of the same length.</span>
  <span class="tok-k">return</span> <span class="tok-p">((</span><span class="tok-n">y_true</span> <span class="tok-o">-</span> <span class="tok-n">y_pred</span><span class="tok-p">)</span> <span class="tok-o">**</span> <span class="tok-mi">2</span><span class="tok-p">)</span><span class="tok-o">.</span><span class="tok-n">mean</span><span class="tok-p">()</span>

<span class="tok-n">y_true</span> <span class="tok-o">=</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">array</span><span class="tok-p">([</span><span class="tok-mi">1</span><span class="tok-p">,</span> <span class="tok-mi">0</span><span class="tok-p">,</span> <span class="tok-mi">0</span><span class="tok-p">,</span> <span class="tok-mi">1</span><span class="tok-p">])</span>
<span class="tok-n">y_pred</span> <span class="tok-o">=</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">array</span><span class="tok-p">([</span><span class="tok-mi">0</span><span class="tok-p">,</span> <span class="tok-mi">0</span><span class="tok-p">,</span> <span class="tok-mi">0</span><span class="tok-p">,</span> <span class="tok-mi">0</span><span class="tok-p">])</span>

<span class="tok-k">print</span><span class="tok-p">(</span><span class="tok-n">mse_loss</span><span class="tok-p">(</span><span class="tok-n">y_true</span><span class="tok-p">,</span> <span class="tok-n">y_pred</span><span class="tok-p">))</span> <span class="tok-c1"># 0.5</span></code></pre>
</div>
</div>
</div>
<div class="sect2">
<h3 id="_训练神经网络第2部分">6.11. 训练神经网络，第2部分</h3>
<div class="paragraph">
<p>现在我们有了一个明确的目标：最小化神经网络的损失。通过调整网络的权重和截距项，我们可以改变其预测结果，但如何才能逐步地减少损失？</p>
</div>
<div class="paragraph">
<p>为了简化问题，假设我们的数据集中只有Alice：</p>
</div>
<table class="tableblock frame-all grid-all stretch">
<colgroup>
<col style="width: 25%;">
<col style="width: 25%;">
<col style="width: 25%;">
<col style="width: 25%;">
</colgroup>
<thead>
<tr>
<th class="tableblock halign-left valign-top">Name</th>
<th class="tableblock halign-left valign-top">\(y_{true}\)</th>
<th class="tableblock halign-left valign-top">\(y_{pred}\)</th>
<th class="tableblock halign-left valign-top">\((y_{true}-y_{pred})^2\)</th>
</tr>
</thead>
<tbody>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">Alice</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">1</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">0</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">1</p></td>
</tr>
</tbody>
</table>
<div class="paragraph">
<p>那均方差损失就只是Alice的方差：</p>
</div>
<div class="stemblock">
<div class="content">
\[\begin{align}  MSE &amp;= \frac{1}{1}\sum_{i=1}^{1}(y_{true} - y_{pred})^2\\  &amp;=(y_{true}-y_{pred})^2\\ &amp;=(1-y_{pred})^2  \end{align}\]
</div>
</div>
<div class="paragraph">
<p>也可以把损失看成是权重和截距项的函数。让我们给网络标上权重和截距项：</p>
</div>
<div class="imageblock">
<div class="content">
<img src="images/dnn5.svg" alt="dnn5">
</div>
</div>
<div class="paragraph">
<p>这样我们就可以把网络的损失表示为：</p>
</div>
<div class="stemblock">
<div class="content">
\[L(w_1, w_2, w_3, w_4, w_5, w_6, b_1, b_2, b_3)\]
</div>
</div>
<div class="paragraph">
<p>假设我们要优化\(w_1\)，当我们改变\(w_1\)时，损失\(L\)会怎么变化？可以用\(\frac{\partial L}{\partial w_1}\)来回答这个问题，怎么计算？</p>
</div>
<div class="admonitionblock note">
<table>
<tr>
<td class="icon">
<i class="fa icon-note" title="Note"></i>
</td>
<td class="content">
接下来的数据稍微有点复杂，别担心，准备好纸和笔。
</td>
</tr>
</table>
</div>
<div class="paragraph">
<p>首先，让我们用\(\frac{\partial y_{pred}}{\partial w_1}\)来改写这个偏导数：</p>
</div>
<div class="stemblock">
<div class="content">
\[\frac{\partial L}{\partial w_1} = \frac{\partial L}{\partial y_{pred}} * \frac{\partial y_{pred}}{\partial w_1}\]
</div>
</div>
<div class="paragraph">
<p>因为我们已经知道 \(L=(1-y_{pred})^2\) ，所以我们可以计算 \(\frac{\partial L}{\partial y_{pred}}\) ：</p>
</div>
<div class="stemblock">
<div class="content">
\[\frac{\partial L}{\partial y_{pred}} = \frac{\partial (1-y_{pred})^2}{\partial y_{pred}}=-2(1-y_{pred})\]
</div>
</div>
<div class="paragraph">
<p>现在让我们来搞定\(\frac{\partial y_{pred}}{\partial w_1}\)。\(h_1,h_2,o_1\)分别是其所表示的神经元的输出，我们有：</p>
</div>
<div class="stemblock">
<div class="content">
\[y_{pred} = o_1 = f(w_5h_1 + w_6h_2 + b_3)\]
</div>
</div>
<div class="paragraph">
<p>由于\(w_1\)只会影响\(h_1\)（不会影响\(h_2\)），所以：</p>
</div>
<div class="stemblock">
<div class="content">
\[ \frac{\partial y_{pred}}{\partial w_1} = \frac{\partial y_{pred}}{\partial h_1} * \frac{\partial h_1}{\partial w_1}\\ \frac{\partial y_{pred}}{\partial h_1} = w_5 * f'(w_5h_1 + w_6h_2 + b_3)\]
</div>
</div>
<div class="paragraph">
<p>对\(\frac{\partial h_1}{\partial w_1}\)，我们也可以这么做：</p>
</div>
<div class="stemblock">
<div class="content">
\[ h_1 = f(w_1x_1 + w_2x_2 + b_1)\\  \frac{\partial h_1}{\partial w_1} = x_1 * f'(w_1x_1 + w_2x_2 + b_1)\]
</div>
</div>
<div class="paragraph">
<p>在这里，\(x_1\) 是身高，\(x_2\)是体重。这是我们第二次看到\(f'(x)\)（S型函数的导数）了。求解：</p>
</div>
<div class="stemblock">
<div class="content">
\[ f(x)=\frac{1}{1-e^{-x}}\\  f'(x)=\frac{e^x}{(1+e^{-x})^2}=f(x)*(1-f(x))\]
</div>
</div>
<div class="paragraph">
<p>稍后我们会用到这个\(f'(x)\)。</p>
</div>
<div class="paragraph">
<p>我们已经把\(\frac{\partial L}{\partial w_1}\)分解成了几个我们能计算的部分：</p>
</div>
<div class="stemblock">
<div class="content">
\[\frac{\partial L}{\partial w_1}= \frac{\partial L}{\partial y_{pred}}\frac{\partial y_{pred}}{\partial h_1}\frac{\partial h_1}{\partial w_1}\]
</div>
</div>
<div class="paragraph">
<p>这种计算偏导的方法叫『反向传播算法』(backpropagation)。</p>
</div>
<div class="paragraph">
<p>好多数学符号，如果你还没搞明白的话，我们来看一个实际例子。</p>
</div>
</div>
<div class="sect2">
<h3 id="_例子计算偏导数">6.12. 例子：计算偏导数</h3>
<div class="paragraph">
<p>我们还是看数据集中只有Alice的情况：</p>
</div>
<table class="tableblock frame-all grid-all stretch">
<colgroup>
<col style="width: 25%;">
<col style="width: 25%;">
<col style="width: 25%;">
<col style="width: 25%;">
</colgroup>
<thead>
<tr>
<th class="tableblock halign-left valign-top">Name</th>
<th class="tableblock halign-left valign-top">\(y_{true}\)</th>
<th class="tableblock halign-left valign-top">\(y_{pred}\)</th>
<th class="tableblock halign-left valign-top">\((y_{true}-y_{pred})^2\)</th>
</tr>
</thead>
<tbody>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">Alice</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">1</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">0</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">1</p></td>
</tr>
</tbody>
</table>
<div class="paragraph">
<p>把所有的权重和截距项都分别初始化为1和0。在网络中做前馈计算：</p>
</div>
<div class="stemblock">
<div class="content">
\[\begin{align}  h_1 &amp;= f(w_1x_1 + w_2x_2 + b_1)\\  &amp;=f(-2 + -1 + 0)\\  &amp;=0.0474 \\   h_2 &amp;= f(w_3x_1 + w_4x_2 + b_2) = 0.0474\\  o_1 &amp;=f(w_5h_1 + w_6h_2 + b_3)\\  &amp;=f(0.0474  + 0.0474 + 0)\\  &amp;=0.524  \end{align}\]
</div>
</div>
<div class="paragraph">
<p>网络的输出是\(y_{pred}=0.524\)，对于Male(0)或者Female(1)都没有太强的倾向性。算一下\(\frac{\partial L}{\partial w_1}\):</p>
</div>
<div class="stemblock">
<div class="content">
\[ \frac{\partial L}{\partial w_1}= \frac{\partial L}{\partial y_{pred}}\frac{\partial y_{pred}}{\partial h_1}\frac{\partial h_1}{\partial w_1}\\ \begin{align*}  \frac{\partial L}{\partial y_{pred}} &amp;= -2(1-y_{pred})\\  &amp; = -2 (1-0.524)\\  &amp;=-0.952\\  \frac{\partial y_{pred}}{\partial h_1} &amp; = x_1 * f'(w_5h_1 + w_6h_2 + b_3)\\  &amp;=1 * f'(0.0474 + 0.0474 + 0)\\  &amp;=f(0.0948)(1-f(0.0948))\\ &amp;=0.249\\  \frac{\partial h_1}{\partial w_1} &amp;= x_1 * f'(w_1x_1 + w_2x_2 + b_1)\\  &amp; = -2 * f'(-2 + -1 + 0)\\  &amp;=-2f(-3)(1-f(-3))\\  &amp;=-0.0904\\  \frac{\partial L}{\partial w_1} &amp;= -0.952 * 0.249 * -0.0904\\ &amp;=0.0214  \end{align*}\]
</div>
</div>
<div class="admonitionblock note">
<table>
<tr>
<td class="icon">
<i class="fa icon-note" title="Note"></i>
</td>
<td class="content">
提示：前面已经得到了S型激活函数的导数\(f'(x)=f(x)*(1-f(x))\)。
</td>
</tr>
</table>
</div>
<div class="paragraph">
<p>搞定！这个结果的意思就是增加\(w_1\)，\(L\)也会随之轻微上升。</p>
</div>
</div>
<div class="sect2">
<h3 id="_训练随机梯度下降">6.13. 训练：随机梯度下降</h3>
<div class="paragraph">
<p>现在训练神经网络已经万事俱备了！我们会使用名为随机梯度下降法的优化算法来优化网络的权重和截距项，实现损失的最小化。核心就是这个更新等式：</p>
</div>
<div class="stemblock">
<div class="content">
\[w_1 \leftarrow w_1 - \eta\frac{\partial L}{\partial w_1}\]
</div>
</div>
<div class="paragraph">
<p>\(\eta\)是一个常数，被称为学习率，用于调整训练的速度。我们要做的就是用\(w_1\)减去\(\eta\frac{\partial L}{\partial w_1}\)：</p>
</div>
<div class="ulist">
<ul>
<li>
<p>如果\(\frac{\partial L}{\partial w_1}\)是正数，\(w_1\)会变小，\(L\)会下降。</p>
</li>
<li>
<p>如果\(\frac{\partial L}{\partial w_1}\)是负数，\(w_1\)会变大，\(L\)会上升。</p>
</li>
</ul>
</div>
<div class="paragraph">
<p>如果我们对网络中的每个权重和截距项都这样进行优化，损失就会不断下降，网络性能会不断上升。</p>
</div>
<div class="paragraph">
<p>我们的训练过程是这样的：</p>
</div>
<div class="olist arabic">
<ol class="arabic">
<li>
<p>从我们的数据集中选择一个样本，用随机梯度下降法进行优化——每次我们都只针对一个样本进行优化；</p>
</li>
<li>
<p>计算每个权重或截距项对损失的偏导（例如\(\frac{\partial L}{\partial w_1}\)、\(\frac{\partial L}{\partial w_2}\)等）；</p>
</li>
<li>
<p>用更新等式更新每个权重和截距项；</p>
</li>
<li>
<p>重复第一步；</p>
</li>
</ol>
</div>
</div>
<div class="sect2">
<h3 id="_代码一个完整的神经网络">6.14. 代码：一个完整的神经网络</h3>
<div class="paragraph">
<p>我们终于可以实现一个完整的神经网络了：</p>
</div>
<table class="tableblock frame-all grid-all stretch">
<colgroup>
<col style="width: 25%;">
<col style="width: 25%;">
<col style="width: 25%;">
<col style="width: 25%;">
</colgroup>
<thead>
<tr>
<th class="tableblock halign-left valign-top">Name</th>
<th class="tableblock halign-left valign-top">Weight</th>
<th class="tableblock halign-left valign-top">Height</th>
<th class="tableblock halign-left valign-top">Gender</th>
</tr>
</thead>
<tbody>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">Alice</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">-2</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">-1</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">1</p></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">Bob</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">25</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">6</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">0</p></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">Charlie</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">17</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">4</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">0</p></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">Diana</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">-15</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">-6</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">1</p></td>
</tr>
</tbody>
</table>
<div class="imageblock">
<div class="content">
<img src="images/dnn6.svg" alt="dnn6">
</div>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="py"><span></span><span class="tok-kn">import</span> <span class="tok-nn">numpy</span> <span class="tok-kn">as</span> <span class="tok-nn">np</span>

<span class="tok-k">def</span> <span class="tok-nf">sigmoid</span><span class="tok-p">(</span><span class="tok-n">x</span><span class="tok-p">):</span>
  <span class="tok-c1"># Sigmoid activation function: f(x) = 1 / (1 + e^(-x))</span>
  <span class="tok-k">return</span> <span class="tok-mi">1</span> <span class="tok-o">/</span> <span class="tok-p">(</span><span class="tok-mi">1</span> <span class="tok-o">+</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">exp</span><span class="tok-p">(</span><span class="tok-o">-</span><span class="tok-n">x</span><span class="tok-p">))</span>

<span class="tok-k">def</span> <span class="tok-nf">deriv_sigmoid</span><span class="tok-p">(</span><span class="tok-n">x</span><span class="tok-p">):</span>
  <span class="tok-c1"># Derivative of sigmoid: f&#39;(x) = f(x) * (1 - f(x))</span>
  <span class="tok-n">fx</span> <span class="tok-o">=</span> <span class="tok-n">sigmoid</span><span class="tok-p">(</span><span class="tok-n">x</span><span class="tok-p">)</span>
  <span class="tok-k">return</span> <span class="tok-n">fx</span> <span class="tok-o">*</span> <span class="tok-p">(</span><span class="tok-mi">1</span> <span class="tok-o">-</span> <span class="tok-n">fx</span><span class="tok-p">)</span>

<span class="tok-k">def</span> <span class="tok-nf">mse_loss</span><span class="tok-p">(</span><span class="tok-n">y_true</span><span class="tok-p">,</span> <span class="tok-n">y_pred</span><span class="tok-p">):</span>
  <span class="tok-c1"># y_true and y_pred are numpy arrays of the same length.</span>
  <span class="tok-k">return</span> <span class="tok-p">((</span><span class="tok-n">y_true</span> <span class="tok-o">-</span> <span class="tok-n">y_pred</span><span class="tok-p">)</span> <span class="tok-o">**</span> <span class="tok-mi">2</span><span class="tok-p">)</span><span class="tok-o">.</span><span class="tok-n">mean</span><span class="tok-p">()</span>

<span class="tok-k">class</span> <span class="tok-nc">OurNeuralNetwork</span><span class="tok-p">:</span>
  <span class="tok-sd">&#39;&#39;&#39;</span>
<span class="tok-sd">  A neural network with:</span>
<span class="tok-sd">    - 2 inputs</span>
<span class="tok-sd">    - a hidden layer with 2 neurons (h1, h2)</span>
<span class="tok-sd">    - an output layer with 1 neuron (o1)</span>

<span class="tok-sd">  *** DISCLAIMER ***:</span>
<span class="tok-sd">  The code below is intended to be simple and educational, NOT optimal.</span>
<span class="tok-sd">  Real neural net code looks nothing like this. DO NOT use this code.</span>
<span class="tok-sd">  Instead, read/run it to understand how this specific network works.</span>
<span class="tok-sd">  &#39;&#39;&#39;</span>
  <span class="tok-k">def</span> <span class="tok-fm">__init__</span><span class="tok-p">(</span><span class="tok-bp">self</span><span class="tok-p">):</span>
    <span class="tok-c1"># Weights</span>
    <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w1</span> <span class="tok-o">=</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">random</span><span class="tok-o">.</span><span class="tok-n">normal</span><span class="tok-p">()</span>
    <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w2</span> <span class="tok-o">=</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">random</span><span class="tok-o">.</span><span class="tok-n">normal</span><span class="tok-p">()</span>
    <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w3</span> <span class="tok-o">=</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">random</span><span class="tok-o">.</span><span class="tok-n">normal</span><span class="tok-p">()</span>
    <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w4</span> <span class="tok-o">=</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">random</span><span class="tok-o">.</span><span class="tok-n">normal</span><span class="tok-p">()</span>
    <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w5</span> <span class="tok-o">=</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">random</span><span class="tok-o">.</span><span class="tok-n">normal</span><span class="tok-p">()</span>
    <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w6</span> <span class="tok-o">=</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">random</span><span class="tok-o">.</span><span class="tok-n">normal</span><span class="tok-p">()</span>

    <span class="tok-c1"># Biases</span>
    <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">b1</span> <span class="tok-o">=</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">random</span><span class="tok-o">.</span><span class="tok-n">normal</span><span class="tok-p">()</span>
    <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">b2</span> <span class="tok-o">=</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">random</span><span class="tok-o">.</span><span class="tok-n">normal</span><span class="tok-p">()</span>
    <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">b3</span> <span class="tok-o">=</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">random</span><span class="tok-o">.</span><span class="tok-n">normal</span><span class="tok-p">()</span>

  <span class="tok-k">def</span> <span class="tok-nf">feedforward</span><span class="tok-p">(</span><span class="tok-bp">self</span><span class="tok-p">,</span> <span class="tok-n">x</span><span class="tok-p">):</span>
    <span class="tok-c1"># x is a numpy array with 2 elements.</span>
    <span class="tok-n">h1</span> <span class="tok-o">=</span> <span class="tok-n">sigmoid</span><span class="tok-p">(</span><span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w1</span> <span class="tok-o">*</span> <span class="tok-n">x</span><span class="tok-p">[</span><span class="tok-mi">0</span><span class="tok-p">]</span> <span class="tok-o">+</span> <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w2</span> <span class="tok-o">*</span> <span class="tok-n">x</span><span class="tok-p">[</span><span class="tok-mi">1</span><span class="tok-p">]</span> <span class="tok-o">+</span> <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">b1</span><span class="tok-p">)</span>
    <span class="tok-n">h2</span> <span class="tok-o">=</span> <span class="tok-n">sigmoid</span><span class="tok-p">(</span><span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w3</span> <span class="tok-o">*</span> <span class="tok-n">x</span><span class="tok-p">[</span><span class="tok-mi">0</span><span class="tok-p">]</span> <span class="tok-o">+</span> <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w4</span> <span class="tok-o">*</span> <span class="tok-n">x</span><span class="tok-p">[</span><span class="tok-mi">1</span><span class="tok-p">]</span> <span class="tok-o">+</span> <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">b2</span><span class="tok-p">)</span>
    <span class="tok-n">o1</span> <span class="tok-o">=</span> <span class="tok-n">sigmoid</span><span class="tok-p">(</span><span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w5</span> <span class="tok-o">*</span> <span class="tok-n">h1</span> <span class="tok-o">+</span> <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w6</span> <span class="tok-o">*</span> <span class="tok-n">h2</span> <span class="tok-o">+</span> <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">b3</span><span class="tok-p">)</span>
    <span class="tok-k">return</span> <span class="tok-n">o1</span>

  <span class="tok-k">def</span> <span class="tok-nf">train</span><span class="tok-p">(</span><span class="tok-bp">self</span><span class="tok-p">,</span> <span class="tok-n">data</span><span class="tok-p">,</span> <span class="tok-n">all_y_trues</span><span class="tok-p">):</span>
    <span class="tok-sd">&#39;&#39;&#39;</span>
<span class="tok-sd">    - data is a (n x 2) numpy array, n = # of samples in the dataset.</span>
<span class="tok-sd">    - all_y_trues is a numpy array with n elements.</span>
<span class="tok-sd">      Elements in all_y_trues correspond to those in data.</span>
<span class="tok-sd">    &#39;&#39;&#39;</span>
    <span class="tok-n">learn_rate</span> <span class="tok-o">=</span> <span class="tok-mf">0.1</span>
    <span class="tok-n">epochs</span> <span class="tok-o">=</span> <span class="tok-mi">1000</span> <span class="tok-c1"># number of times to loop through the entire dataset</span>

    <span class="tok-k">for</span> <span class="tok-n">epoch</span> <span class="tok-ow">in</span> <span class="tok-nb">range</span><span class="tok-p">(</span><span class="tok-n">epochs</span><span class="tok-p">):</span>
      <span class="tok-k">for</span> <span class="tok-n">x</span><span class="tok-p">,</span> <span class="tok-n">y_true</span> <span class="tok-ow">in</span> <span class="tok-nb">zip</span><span class="tok-p">(</span><span class="tok-n">data</span><span class="tok-p">,</span> <span class="tok-n">all_y_trues</span><span class="tok-p">):</span>
        <span class="tok-c1"># --- Do a feedforward (we&#39;ll need these values later)</span>
        <span class="tok-n">sum_h1</span> <span class="tok-o">=</span> <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w1</span> <span class="tok-o">*</span> <span class="tok-n">x</span><span class="tok-p">[</span><span class="tok-mi">0</span><span class="tok-p">]</span> <span class="tok-o">+</span> <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w2</span> <span class="tok-o">*</span> <span class="tok-n">x</span><span class="tok-p">[</span><span class="tok-mi">1</span><span class="tok-p">]</span> <span class="tok-o">+</span> <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">b1</span>
        <span class="tok-n">h1</span> <span class="tok-o">=</span> <span class="tok-n">sigmoid</span><span class="tok-p">(</span><span class="tok-n">sum_h1</span><span class="tok-p">)</span>

        <span class="tok-n">sum_h2</span> <span class="tok-o">=</span> <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w3</span> <span class="tok-o">*</span> <span class="tok-n">x</span><span class="tok-p">[</span><span class="tok-mi">0</span><span class="tok-p">]</span> <span class="tok-o">+</span> <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w4</span> <span class="tok-o">*</span> <span class="tok-n">x</span><span class="tok-p">[</span><span class="tok-mi">1</span><span class="tok-p">]</span> <span class="tok-o">+</span> <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">b2</span>
        <span class="tok-n">h2</span> <span class="tok-o">=</span> <span class="tok-n">sigmoid</span><span class="tok-p">(</span><span class="tok-n">sum_h2</span><span class="tok-p">)</span>

        <span class="tok-n">sum_o1</span> <span class="tok-o">=</span> <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w5</span> <span class="tok-o">*</span> <span class="tok-n">h1</span> <span class="tok-o">+</span> <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w6</span> <span class="tok-o">*</span> <span class="tok-n">h2</span> <span class="tok-o">+</span> <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">b3</span>
        <span class="tok-n">o1</span> <span class="tok-o">=</span> <span class="tok-n">sigmoid</span><span class="tok-p">(</span><span class="tok-n">sum_o1</span><span class="tok-p">)</span>
        <span class="tok-n">y_pred</span> <span class="tok-o">=</span> <span class="tok-n">o1</span>

        <span class="tok-c1"># --- Calculate partial derivatives.</span>
        <span class="tok-c1"># --- Naming: d_L_d_w1 represents &quot;partial L / partial w1&quot;</span>
        <span class="tok-n">d_L_d_ypred</span> <span class="tok-o">=</span> <span class="tok-o">-</span><span class="tok-mi">2</span> <span class="tok-o">*</span> <span class="tok-p">(</span><span class="tok-n">y_true</span> <span class="tok-o">-</span> <span class="tok-n">y_pred</span><span class="tok-p">)</span>

        <span class="tok-c1"># Neuron o1</span>
        <span class="tok-n">d_ypred_d_w5</span> <span class="tok-o">=</span> <span class="tok-n">h1</span> <span class="tok-o">*</span> <span class="tok-n">deriv_sigmoid</span><span class="tok-p">(</span><span class="tok-n">sum_o1</span><span class="tok-p">)</span>
        <span class="tok-n">d_ypred_d_w6</span> <span class="tok-o">=</span> <span class="tok-n">h2</span> <span class="tok-o">*</span> <span class="tok-n">deriv_sigmoid</span><span class="tok-p">(</span><span class="tok-n">sum_o1</span><span class="tok-p">)</span>
        <span class="tok-n">d_ypred_d_b3</span> <span class="tok-o">=</span> <span class="tok-n">deriv_sigmoid</span><span class="tok-p">(</span><span class="tok-n">sum_o1</span><span class="tok-p">)</span>

        <span class="tok-n">d_ypred_d_h1</span> <span class="tok-o">=</span> <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w5</span> <span class="tok-o">*</span> <span class="tok-n">deriv_sigmoid</span><span class="tok-p">(</span><span class="tok-n">sum_o1</span><span class="tok-p">)</span>
        <span class="tok-n">d_ypred_d_h2</span> <span class="tok-o">=</span> <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w6</span> <span class="tok-o">*</span> <span class="tok-n">deriv_sigmoid</span><span class="tok-p">(</span><span class="tok-n">sum_o1</span><span class="tok-p">)</span>

        <span class="tok-c1"># Neuron h1</span>
        <span class="tok-n">d_h1_d_w1</span> <span class="tok-o">=</span> <span class="tok-n">x</span><span class="tok-p">[</span><span class="tok-mi">0</span><span class="tok-p">]</span> <span class="tok-o">*</span> <span class="tok-n">deriv_sigmoid</span><span class="tok-p">(</span><span class="tok-n">sum_h1</span><span class="tok-p">)</span>
        <span class="tok-n">d_h1_d_w2</span> <span class="tok-o">=</span> <span class="tok-n">x</span><span class="tok-p">[</span><span class="tok-mi">1</span><span class="tok-p">]</span> <span class="tok-o">*</span> <span class="tok-n">deriv_sigmoid</span><span class="tok-p">(</span><span class="tok-n">sum_h1</span><span class="tok-p">)</span>
        <span class="tok-n">d_h1_d_b1</span> <span class="tok-o">=</span> <span class="tok-n">deriv_sigmoid</span><span class="tok-p">(</span><span class="tok-n">sum_h1</span><span class="tok-p">)</span>

        <span class="tok-c1"># Neuron h2</span>
        <span class="tok-n">d_h2_d_w3</span> <span class="tok-o">=</span> <span class="tok-n">x</span><span class="tok-p">[</span><span class="tok-mi">0</span><span class="tok-p">]</span> <span class="tok-o">*</span> <span class="tok-n">deriv_sigmoid</span><span class="tok-p">(</span><span class="tok-n">sum_h2</span><span class="tok-p">)</span>
        <span class="tok-n">d_h2_d_w4</span> <span class="tok-o">=</span> <span class="tok-n">x</span><span class="tok-p">[</span><span class="tok-mi">1</span><span class="tok-p">]</span> <span class="tok-o">*</span> <span class="tok-n">deriv_sigmoid</span><span class="tok-p">(</span><span class="tok-n">sum_h2</span><span class="tok-p">)</span>
        <span class="tok-n">d_h2_d_b2</span> <span class="tok-o">=</span> <span class="tok-n">deriv_sigmoid</span><span class="tok-p">(</span><span class="tok-n">sum_h2</span><span class="tok-p">)</span>

        <span class="tok-c1"># --- Update weights and biases</span>
        <span class="tok-c1"># Neuron h1</span>
        <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w1</span> <span class="tok-o">-=</span> <span class="tok-n">learn_rate</span> <span class="tok-o">*</span> <span class="tok-n">d_L_d_ypred</span> <span class="tok-o">*</span> <span class="tok-n">d_ypred_d_h1</span> <span class="tok-o">*</span> <span class="tok-n">d_h1_d_w1</span>
        <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w2</span> <span class="tok-o">-=</span> <span class="tok-n">learn_rate</span> <span class="tok-o">*</span> <span class="tok-n">d_L_d_ypred</span> <span class="tok-o">*</span> <span class="tok-n">d_ypred_d_h1</span> <span class="tok-o">*</span> <span class="tok-n">d_h1_d_w2</span>
        <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">b1</span> <span class="tok-o">-=</span> <span class="tok-n">learn_rate</span> <span class="tok-o">*</span> <span class="tok-n">d_L_d_ypred</span> <span class="tok-o">*</span> <span class="tok-n">d_ypred_d_h1</span> <span class="tok-o">*</span> <span class="tok-n">d_h1_d_b1</span>

        <span class="tok-c1"># Neuron h2</span>
        <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w3</span> <span class="tok-o">-=</span> <span class="tok-n">learn_rate</span> <span class="tok-o">*</span> <span class="tok-n">d_L_d_ypred</span> <span class="tok-o">*</span> <span class="tok-n">d_ypred_d_h2</span> <span class="tok-o">*</span> <span class="tok-n">d_h2_d_w3</span>
        <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w4</span> <span class="tok-o">-=</span> <span class="tok-n">learn_rate</span> <span class="tok-o">*</span> <span class="tok-n">d_L_d_ypred</span> <span class="tok-o">*</span> <span class="tok-n">d_ypred_d_h2</span> <span class="tok-o">*</span> <span class="tok-n">d_h2_d_w4</span>
        <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">b2</span> <span class="tok-o">-=</span> <span class="tok-n">learn_rate</span> <span class="tok-o">*</span> <span class="tok-n">d_L_d_ypred</span> <span class="tok-o">*</span> <span class="tok-n">d_ypred_d_h2</span> <span class="tok-o">*</span> <span class="tok-n">d_h2_d_b2</span>

        <span class="tok-c1"># Neuron o1</span>
        <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w5</span> <span class="tok-o">-=</span> <span class="tok-n">learn_rate</span> <span class="tok-o">*</span> <span class="tok-n">d_L_d_ypred</span> <span class="tok-o">*</span> <span class="tok-n">d_ypred_d_w5</span>
        <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">w6</span> <span class="tok-o">-=</span> <span class="tok-n">learn_rate</span> <span class="tok-o">*</span> <span class="tok-n">d_L_d_ypred</span> <span class="tok-o">*</span> <span class="tok-n">d_ypred_d_w6</span>
        <span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">b3</span> <span class="tok-o">-=</span> <span class="tok-n">learn_rate</span> <span class="tok-o">*</span> <span class="tok-n">d_L_d_ypred</span> <span class="tok-o">*</span> <span class="tok-n">d_ypred_d_b3</span>

      <span class="tok-c1"># --- Calculate total loss at the end of each epoch</span>
      <span class="tok-k">if</span> <span class="tok-n">epoch</span> <span class="tok-o">%</span> <span class="tok-mi">10</span> <span class="tok-o">==</span> <span class="tok-mi">0</span><span class="tok-p">:</span>
        <span class="tok-n">y_preds</span> <span class="tok-o">=</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">apply_along_axis</span><span class="tok-p">(</span><span class="tok-bp">self</span><span class="tok-o">.</span><span class="tok-n">feedforward</span><span class="tok-p">,</span> <span class="tok-mi">1</span><span class="tok-p">,</span> <span class="tok-n">data</span><span class="tok-p">)</span>
        <span class="tok-n">loss</span> <span class="tok-o">=</span> <span class="tok-n">mse_loss</span><span class="tok-p">(</span><span class="tok-n">all_y_trues</span><span class="tok-p">,</span> <span class="tok-n">y_preds</span><span class="tok-p">)</span>
        <span class="tok-k">print</span><span class="tok-p">(</span><span class="tok-s2">&quot;Epoch </span><span class="tok-si">%d</span><span class="tok-s2"> loss: </span><span class="tok-si">%.3f</span><span class="tok-s2">&quot;</span> <span class="tok-o">%</span> <span class="tok-p">(</span><span class="tok-n">epoch</span><span class="tok-p">,</span> <span class="tok-n">loss</span><span class="tok-p">))</span>

<span class="tok-c1"># Define dataset</span>
<span class="tok-n">data</span> <span class="tok-o">=</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">array</span><span class="tok-p">([</span>
  <span class="tok-p">[</span><span class="tok-o">-</span><span class="tok-mi">2</span><span class="tok-p">,</span> <span class="tok-o">-</span><span class="tok-mi">1</span><span class="tok-p">],</span>  <span class="tok-c1"># Alice</span>
  <span class="tok-p">[</span><span class="tok-mi">25</span><span class="tok-p">,</span> <span class="tok-mi">6</span><span class="tok-p">],</span>   <span class="tok-c1"># Bob</span>
  <span class="tok-p">[</span><span class="tok-mi">17</span><span class="tok-p">,</span> <span class="tok-mi">4</span><span class="tok-p">],</span>   <span class="tok-c1"># Charlie</span>
  <span class="tok-p">[</span><span class="tok-o">-</span><span class="tok-mi">15</span><span class="tok-p">,</span> <span class="tok-o">-</span><span class="tok-mi">6</span><span class="tok-p">],</span> <span class="tok-c1"># Diana</span>
<span class="tok-p">])</span>
<span class="tok-n">all_y_trues</span> <span class="tok-o">=</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">array</span><span class="tok-p">([</span>
  <span class="tok-mi">1</span><span class="tok-p">,</span> <span class="tok-c1"># Alice</span>
  <span class="tok-mi">0</span><span class="tok-p">,</span> <span class="tok-c1"># Bob</span>
  <span class="tok-mi">0</span><span class="tok-p">,</span> <span class="tok-c1"># Charlie</span>
  <span class="tok-mi">1</span><span class="tok-p">,</span> <span class="tok-c1"># Diana</span>
<span class="tok-p">])</span>

<span class="tok-c1"># Train our neural network!</span>
<span class="tok-n">network</span> <span class="tok-o">=</span> <span class="tok-n">OurNeuralNetwork</span><span class="tok-p">()</span>
<span class="tok-n">network</span><span class="tok-o">.</span><span class="tok-n">train</span><span class="tok-p">(</span><span class="tok-n">data</span><span class="tok-p">,</span> <span class="tok-n">all_y_trues</span><span class="tok-p">)</span></code></pre>
</div>
</div>
<div class="paragraph">
<p>随着网络的学习，损失在稳步下降。</p>
</div>
<div class="imageblock">
<div class="content">
<img src="images/dnn7.png" alt="dnn7">
</div>
</div>
<div class="paragraph">
<p>现在我们可以用这个网络来预测性别了：</p>
</div>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="py"><span></span><span class="tok-c1"># Make some predictions</span>
<span class="tok-n">emily</span> <span class="tok-o">=</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">array</span><span class="tok-p">([</span><span class="tok-o">-</span><span class="tok-mi">7</span><span class="tok-p">,</span> <span class="tok-o">-</span><span class="tok-mi">3</span><span class="tok-p">])</span> <span class="tok-c1"># 128 pounds, 63 inches</span>
<span class="tok-n">frank</span> <span class="tok-o">=</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">array</span><span class="tok-p">([</span><span class="tok-mi">20</span><span class="tok-p">,</span> <span class="tok-mi">2</span><span class="tok-p">])</span>  <span class="tok-c1"># 155 pounds, 68 inches</span>
<span class="tok-k">print</span><span class="tok-p">(</span><span class="tok-s2">&quot;Emily: </span><span class="tok-si">%.3f</span><span class="tok-s2">&quot;</span> <span class="tok-o">%</span> <span class="tok-n">network</span><span class="tok-o">.</span><span class="tok-n">feedforward</span><span class="tok-p">(</span><span class="tok-n">emily</span><span class="tok-p">))</span> <span class="tok-c1"># 0.951 - F</span>
<span class="tok-k">print</span><span class="tok-p">(</span><span class="tok-s2">&quot;Frank: </span><span class="tok-si">%.3f</span><span class="tok-s2">&quot;</span> <span class="tok-o">%</span> <span class="tok-n">network</span><span class="tok-o">.</span><span class="tok-n">feedforward</span><span class="tok-p">(</span><span class="tok-n">frank</span><span class="tok-p">))</span> <span class="tok-c1"># 0.039 - M</span></code></pre>
</div>
</div>
</div>
<div class="sect2">
<h3 id="_接下来">6.15. 接下来？</h3>
<div class="paragraph">
<p>搞定了一个简单的神经网络，快速回顾一下：</p>
</div>
<div class="ulist">
<ul>
<li>
<p>介绍了神经网络的基本结构——神经元；</p>
</li>
<li>
<p>在神经元中使用S型激活函数；</p>
</li>
<li>
<p>神经网络就是连接在一起的神经元；</p>
</li>
<li>
<p>构建了一个数据集，输入（或特征）是体重和身高，输出（或标签）是性别；</p>
</li>
<li>
<p>学习了损失函数和均方差损失；</p>
</li>
<li>
<p>训练网络就是最小化其损失；</p>
</li>
<li>
<p>用反向传播方法计算偏导；</p>
</li>
<li>
<p>用随机梯度下降法训练网络；</p>
</li>
</ul>
</div>
<div class="paragraph">
<p>接下来你还可以：</p>
</div>
<div class="ulist">
<ul>
<li>
<p>用机器学习库实现更大更好的神经网络，例如TensorFlow、Keras和PyTorch；</p>
</li>
<li>
<p>在浏览器中实现神经网络；</p>
</li>
<li>
<p>其他类型的激活函数；</p>
</li>
<li>
<p>其他类型的优化器；</p>
</li>
<li>
<p>学习卷积神经网络，这给计算机视觉领域带来了革命；</p>
</li>
<li>
<p>学习递归神经网络，常用语自然语言处理；</p>
</li>
</ul>
</div>
</div>
<div class="sect2">
<h3 id="_tensorflow版本">6.16. TensorFlow版本</h3>
<div class="listingblock">
<div class="content">
<pre class="pygments highlight"><code data-lang="py"><span></span><span class="tok-kn">import</span> <span class="tok-nn">tensorflow</span> <span class="tok-kn">as</span> <span class="tok-nn">tf</span>
<span class="tok-kn">import</span> <span class="tok-nn">numpy</span> <span class="tok-kn">as</span> <span class="tok-nn">np</span>

<span class="tok-n">data</span> <span class="tok-o">=</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">array</span><span class="tok-p">([</span>
  <span class="tok-p">[</span><span class="tok-o">-</span><span class="tok-mf">2.0</span><span class="tok-p">,</span> <span class="tok-o">-</span><span class="tok-mi">1</span><span class="tok-p">],</span>  <span class="tok-c1"># Alice</span>
  <span class="tok-p">[</span><span class="tok-mi">25</span><span class="tok-p">,</span> <span class="tok-mi">6</span><span class="tok-p">],</span>   <span class="tok-c1"># Bob</span>
  <span class="tok-p">[</span><span class="tok-mi">17</span><span class="tok-p">,</span> <span class="tok-mi">4</span><span class="tok-p">],</span>   <span class="tok-c1"># Charlie</span>
  <span class="tok-p">[</span><span class="tok-o">-</span><span class="tok-mi">15</span><span class="tok-p">,</span> <span class="tok-o">-</span><span class="tok-mi">6</span><span class="tok-p">],</span> <span class="tok-c1"># Diana</span>
<span class="tok-p">])</span>
<span class="tok-n">all_y_trues</span> <span class="tok-o">=</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">array</span><span class="tok-p">([</span>
  <span class="tok-mi">1</span><span class="tok-p">,</span> <span class="tok-c1"># Alice</span>
  <span class="tok-mi">0</span><span class="tok-p">,</span> <span class="tok-c1"># Bob</span>
  <span class="tok-mi">0</span><span class="tok-p">,</span> <span class="tok-c1"># Charlie</span>
  <span class="tok-mi">1</span><span class="tok-p">,</span> <span class="tok-c1"># Diana</span>
<span class="tok-p">])</span>

<span class="tok-n">inputs</span> <span class="tok-o">=</span> <span class="tok-n">tf</span><span class="tok-o">.</span><span class="tok-n">keras</span><span class="tok-o">.</span><span class="tok-n">Input</span><span class="tok-p">(</span><span class="tok-n">shape</span><span class="tok-o">=</span><span class="tok-p">(</span><span class="tok-mi">2</span><span class="tok-p">,))</span>
<span class="tok-n">x</span> <span class="tok-o">=</span> <span class="tok-n">tf</span><span class="tok-o">.</span><span class="tok-n">keras</span><span class="tok-o">.</span><span class="tok-n">layers</span><span class="tok-o">.</span><span class="tok-n">Dense</span><span class="tok-p">(</span><span class="tok-mi">2</span><span class="tok-p">,</span> <span class="tok-n">use_bias</span><span class="tok-o">=</span><span class="tok-bp">True</span><span class="tok-p">)(</span><span class="tok-n">inputs</span><span class="tok-p">)</span>
<span class="tok-n">outputs</span> <span class="tok-o">=</span> <span class="tok-n">tf</span><span class="tok-o">.</span><span class="tok-n">keras</span><span class="tok-o">.</span><span class="tok-n">layers</span><span class="tok-o">.</span><span class="tok-n">Dense</span><span class="tok-p">(</span><span class="tok-mi">1</span><span class="tok-p">,</span> <span class="tok-n">use_bias</span><span class="tok-o">=</span><span class="tok-bp">True</span><span class="tok-p">,</span> <span class="tok-n">activation</span><span class="tok-o">=</span><span class="tok-s1">&#39;sigmoid&#39;</span><span class="tok-p">)(</span><span class="tok-n">x</span><span class="tok-p">)</span>
<span class="tok-n">m</span> <span class="tok-o">=</span> <span class="tok-n">tf</span><span class="tok-o">.</span><span class="tok-n">keras</span><span class="tok-o">.</span><span class="tok-n">Model</span><span class="tok-p">(</span><span class="tok-n">inputs</span><span class="tok-p">,</span> <span class="tok-n">outputs</span><span class="tok-p">)</span>

<span class="tok-n">m</span><span class="tok-o">.</span><span class="tok-n">compile</span><span class="tok-p">(</span><span class="tok-n">tf</span><span class="tok-o">.</span><span class="tok-n">keras</span><span class="tok-o">.</span><span class="tok-n">optimizers</span><span class="tok-o">.</span><span class="tok-n">SGD</span><span class="tok-p">(</span><span class="tok-n">learning_rate</span><span class="tok-o">=</span><span class="tok-mf">0.1</span><span class="tok-p">),</span> <span class="tok-s1">&#39;mse&#39;</span><span class="tok-p">)</span>
<span class="tok-n">m</span><span class="tok-o">.</span><span class="tok-n">fit</span><span class="tok-p">(</span><span class="tok-n">data</span><span class="tok-p">,</span> <span class="tok-n">all_y_trues</span><span class="tok-p">,</span> <span class="tok-n">epochs</span><span class="tok-o">=</span><span class="tok-mi">1000</span><span class="tok-p">,</span> <span class="tok-n">batch_size</span><span class="tok-o">=</span><span class="tok-mi">1</span><span class="tok-p">,</span> <span class="tok-n">verbose</span><span class="tok-o">=</span><span class="tok-mi">0</span><span class="tok-p">)</span>

<span class="tok-n">emily</span> <span class="tok-o">=</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">array</span><span class="tok-p">([[</span><span class="tok-o">-</span><span class="tok-mi">7</span><span class="tok-p">,</span> <span class="tok-o">-</span><span class="tok-mi">3</span><span class="tok-p">]])</span>
<span class="tok-n">frank</span> <span class="tok-o">=</span> <span class="tok-n">np</span><span class="tok-o">.</span><span class="tok-n">array</span><span class="tok-p">([[</span><span class="tok-mi">20</span><span class="tok-p">,</span> <span class="tok-mi">2</span><span class="tok-p">]])</span>
<span class="tok-k">print</span><span class="tok-p">(</span><span class="tok-n">m</span><span class="tok-o">.</span><span class="tok-n">predict</span><span class="tok-p">(</span><span class="tok-n">emily</span><span class="tok-p">))</span>
<span class="tok-k">print</span><span class="tok-p">(</span><span class="tok-n">m</span><span class="tok-o">.</span><span class="tok-n">predict</span><span class="tok-p">(</span><span class="tok-n">frank</span><span class="tok-p">))</span></code></pre>
</div>
</div>
</div>
</div>
</div>
<div class="sect1">
<h2 id="_神经网络bp反向传播算法原理和详细推导流程">7. 神经网络BP反向传播算法原理和详细推导流程</h2>
<div class="sectionbody">
<div class="sect2">
<h3 id="_反向传播算法和bp网络简介">7.1. 反向传播算法和BP网络简介</h3>
<div class="paragraph">
<p>误差反向传播算法简称反向传播算法（即BP算法）。使用反向传播算法的多层感知器又称为BP神经网络。BP算法是一个迭代算法，它的基本思想为：</p>
</div>
<div class="olist arabic">
<ol class="arabic">
<li>
<p>先计算每一层的状态和激活值，直到最后一层（即信号是前向传播的）；</p>
</li>
<li>
<p>计算每一层的误差，误差的计算过程是从最后一层向前推进的（这就是反向传播算法名字的由来）；</p>
</li>
<li>
<p>更新参数（目标是误差变小）。迭代前面两个步骤，直到满足停止准则（比如相邻两次迭代的误差的差别很小）。</p>
</li>
</ol>
</div>
<div class="paragraph">
<p>本文的记号说明：</p>
</div>
<div class="ulist">
<ul>
<li>
<p>\(n_l\)表示第\(l\)层神经元的个数；</p>
</li>
<li>
<p>\(f(\cdot)\)表示神经元的激活函数；</p>
</li>
<li>
<p>\(W^{(l)} \in \mathbb{R}^{n_l \times n_{l-1}}\)表示第\(l-1\)层到第\(l\)层的权重矩阵；</p>
</li>
<li>
<p>\(w_{ij}^{(l)}\)是权重矩阵\(W^{(l)}\)中的元素，表示第\(l-1\)层第\(j\)个神经元到第\(l\)层第\(i\)个神经元的连接的权重（注意标号的顺序）；</p>
</li>
<li>
<p>\(\textbf{b}^{(l)}=(b_1^{(l)},b_2^{(l)},\dots,b_{n_l}^{(l)})^T \in \mathbb{R}^{n_l}\)表示\(l-1\)层到第\(l\)层的偏置；</p>
</li>
<li>
<p>\(\textbf{z}^{(l)}=(z_1^{(l)},z_2^{(l)},\dots,z_{n_l}^{(l)})^T \in \mathbb{R}^{n_l}\)表示\(l\)层神经元的状态；</p>
</li>
<li>
<p>\(\textbf{a}^{(l)}=(a_1^{(l)},a_2^{(l)},\dots,a_{n_l}^{(l)})^T \in \mathbb{R}^{n_l}\)表示\(l\)层神经元的激活值（即输出值）；</p>
</li>
</ul>
</div>
<div class="paragraph">
<p><strong>关于记号的特别注意</strong>：不同的文献所采用的记号可能不同，这将导致不同文献的公式结论可能不同。如Andrew Ng的教程中的\(W^{(l)}\)表示的是第\(l\)层到第\(l+1\)层的权重矩阵。又如，本文用“下标”来标记一个向量的不同分量，而有一些资料却用“上标”来标记向量的不同分量。</p>
</div>
<div class="paragraph">
<p>下面以三层感知器(即只含有一个隐藏层的多层感知器)为例介绍“反向传播算法(BP算法)”。</p>
</div>
<div class="paragraph">
<p>三层感知机如图1所示。例子中，输入数据\(\textbf{x}=(x_1,x_2,x_3)^T\)是3维的（对于第一层，可以认为\(a_i^{(1)}=x_i\)），唯一的隐藏层有3个节点，输出数据是2维的。</p>
</div>
<div class="imageblock">
<div class="content">
<img src="images/dnn8.png" alt="dnn8">
</div>
</div>
</div>
<div class="sect2">
<h3 id="_信息前向传播">7.2. 信息前向传播</h3>
<div class="paragraph">
<p>显然，图1所示神经网络的第2层神经元的状态及激活值可以通过下面的计算得到：</p>
</div>
<div class="stemblock">
<div class="content">
\[z_1^{(2)}=w_{11}^{(2)}x_1+w_{12}^{(2)}x_2+w_{13}^{(2)}x_3+b_1^{(2)} \\
z_2^{(2)}=w_{21}^{(2)}x_1+w_{22}^{(2)}x_2+w_{23}^{(2)}x_3+b_2^{(2)} \\
z_3^{(2)}=w_{31}^{(2)}x_1+w_{32}^{(2)}x_2+w_{33}^{(2)}x_3+b_3^{(2)} \\
a_1^{(2)}=f(z_1^{(2)}) \\
a_2^{(2)}=f(z_2^{(2)}) \\
a_3^{(2)}=f(z_3^{(2)})\]
</div>
</div>
<div class="paragraph">
<p>类似的，第3层神经元的状态及激活值可以通过下面的计算得到：</p>
</div>
<div class="stemblock">
<div class="content">
\[z_1^{(3)}=w_{11}^{(3)}a_1^{(2)}+w_{12}^{(2)}a_2^{(2)}+w_{13}^{(2)}a_3^{(2)}+b_1^{(3)} \\
z_2^{(3)}=w_{21}^{(3)}a_1^{(2)}+w_{22}^{(3)}a_2^{(2)}+w_{23}^{(2)}a_3^{(2)}+b_2^{(3)} \\
a_1^{(3)}=f(z_1^{(3)}) \\
a_2^{(3)}=f(z_2^{(3)})\]
</div>
</div>
<div class="paragraph">
<p>可以总结出，第\(l(2 \le l \le L)\)层神经元的状态及激活值为（下面式子是向量表示形式）：</p>
</div>
<div class="stemblock">
<div class="content">
\[\textbf{z}^{(l)}=W^{(l)} \textbf{a}^{(l-1)}+\textbf{b}^{(l)} \\
\textbf{a}^{(l)}=f(\textbf{z}^{(l)})\]
</div>
</div>
<div class="paragraph">
<p>对于\(L\)层感知机，网络的最终输出为\(\textbf{a}^{(L)}\)。前馈神经网络中信息的前向传递过程如下：</p>
</div>
<div class="stemblock">
<div class="content">
\[\textbf{x}=\textbf{a}^{(1)} \rightarrow \textbf{z}^{(2)} \rightarrow \dots \rightarrow \textbf{a}^{(L-1)} \rightarrow \textbf{z}^{(L)} \rightarrow \textbf{a}^{(L)}=y\]
</div>
</div>
</div>
<div class="sect2">
<h3 id="_误差反向传播">7.3. 误差反向传播</h3>
<div class="paragraph">
<p>“信息向前传播”讲的是已知各个神经元的参数后，如何得到神经网络的输出。但怎么得到各个神经元的参数呢？“误差反向传播”算法解决的就是这个问题。</p>
</div>
<div class="paragraph">
<p>假设训练数据为\({(x^{(1)},y^{(1)}),(x^{(2)},y^{(2)}),\dots,(x^{(i)},y^{(i)}),\dots,(x^{(N)},y^{(N)})}\)，即共有\(N\)个。又假设输出数据为\(n_L\)维的，即\(\textbf{y}^{(i)}=(y_1^{(i)},\dots,y_{n_L}^{(i)})^T\)。</p>
</div>
<div class="paragraph">
<p>对某一个训练数据\((\textbf{x}^{(i)},\textbf{y}^{(i)})\)来说，其代价函数可写为：</p>
</div>
<div class="stemblock">
<div class="content">
\[\begin{align}
E_{(i)} &amp;= \frac{1}{2} \parallel \textbf{y}^{(i)} - \textbf{(o)}^{(i)} \parallel \\
&amp;= \frac{1}{2} \sum_{k=1}^{n_L}(y_k^{(i)} - o_k^{(i)})^2
\end{align}\]
</div>
</div>
<div class="paragraph">
<p>说明1：\(\textbf{y}^{(i)}\)为期望的输出（是训练数据给出的已知值），\(\textbf{o}^{(i)}\)为神经网络对输入\(\textbf{x}^{(i)}\)产生的实际输出。</p>
</div>
<div class="paragraph">
<p>说明2：代价函数中的系数\(\frac{1}{2}\)显然是不必要的，它的存在仅仅是为了后续计算时更方便。</p>
</div>
<div class="paragraph">
<p>说明3：以图1所示的神经网络为例子，\(n_L=2,\textbf{y}^{(i)}=(y_1^{(i)},y_2^{(i)})^T\)，从而有\(E_{(i)}=\frac{1}{2}(y_1^{(i)}-a_1^{(3)})^2+\frac{1}{2}(y_2^{(i)}-a_2^{(3)})^2\)，如果展开到隐藏层，则有\(E_{(i)}=\frac{1}{2}(y_1^{(i)}-f(w_{11}^{(3)}a_1^{(2)}+w_{12}^{(3)}a_2^{(2)}+w_{13}^{(3)}a_3^{(2)}+b_1^{(3)}))^2 + \frac{1}{2}(y_2^{(i)}-f(w_{21}^{(3)}a_1^{(2)}+w_{22}^{(3)}a_2^{(2)}+w_{23}^{(3)}a_3^{(2)}+b_2^{(3)}))^2\)，还可以进一步展开到输入层（替换掉\(a_1^{(2)},a_2^{(2)},a_3^{(2)}\)即可），最后可得：<strong>代价函数\(E_{(i)}\)仅和权重矩阵\(W^{(l)}\)和偏置向量\(b^{(l)}\)相关，调整权重和偏置可以减少或增大代价（误差）。</strong></p>
</div>
<div class="paragraph">
<p>显然，所有训练数据的总体（平均）代价可写为：</p>
</div>
<div class="stemblock">
<div class="content">
\[E_{total}=\frac{1}{N}\sum_{i=1}^N E_{(i)}\]
</div>
</div>
<div class="paragraph">
<p>我们的目标就是调整权重和偏置使总体代价（误差）变小，求得总体代价取最小值时对应的各个神经元的参数（即权重和偏置）。</p>
</div>
<div class="paragraph">
<p>如果采用梯度下降法（在这里，又可称为“批量梯度下降法”），可以用下面公式更新参数\(w_{ij}^{(l)},b_i^{(l)},2 \le l \le L\)：</p>
</div>
<div class="stemblock">
<div class="content">
\[\begin{align}
W^{(l)} &amp;= W^{(l)} - \mu\frac{\partial E_{total}}{\partial W^{(l)}} \\
&amp;= W^{(l)} - \frac{\mu}{N}\sum_{i=1}^N \frac{\partial E_{(i)}}{\partial W^{(l)}} \\
b^{(l)} &amp;= b^{(l)} - \mu\frac{\partial E_{total}}{\partial b^{(l)}} \\
&amp;= b^{(l)} - \frac{\mu}{N}\sum_{i=1}^N\frac{\partial E_{(i)}}{\partial b^{(l)}}
\end{align}\]
</div>
</div>
<div class="paragraph">
<p>由上面公式可知，只需求得每一个训练数据的代价函数\(E_{(i)}\)对参数的偏导数\(\frac{\partial E_{(i)}}{\partial W^{(l)}},\frac{\partial E_{(i)}}{\partial b^{(l)}}\)即可得到参数的迭代更新公式。</p>
</div>
<div class="paragraph">
<p><strong>为简单起见，在下文的推导中，我们去掉\(E_{(i)}\)的下标，直接记为E（要理解它是单个训练数据的误差）。</strong></p>
</div>
<div class="paragraph">
<p>下面将介绍用“反向传播算法”求解单个训练数据误差对参数的偏导数\(\frac{\partial E}{\partial W^{(l)}}\)和\(\frac{\partial E}{\partial b^{(l)}}\)的过程。我们求解一个简单的情况：图1所示神经网络，最后再归纳出通用公式。</p>
</div>
<div class="sect3">
<h4 id="_输出层的权重参数更新">7.3.1. 输出层的权重参数更新</h4>
<div class="paragraph">
<p>把\(E\)展开到隐藏层，有：</p>
</div>
<div class="stemblock">
<div class="content">
\[\begin{align}
E &amp;= \frac{1}{2} \parallel \textbf{y}-\textbf{o} \parallel \\
&amp;= \frac{1}{2} \parallel \textbf{y}-\textbf{a}^{(3)} \parallel \\
&amp;= \frac{1}{2}(y_1^{(i)}-a_1^{(3)})^2+\frac{1}{2}(y_2^{(i)}-a_2^{(3)})^2 \\
&amp;= \frac{1}{2}((y_1-f(z_1^{(3)}))^2+(y_2-f(z_2^{(3)}))^2) \\
&amp;= \frac{1}{2}(y_1^{(i)}-f(w_{11}^{(3)}a_1^{(2)}+w_{12}^{(3)}a_2^{(2)}+w_{13}^{(3)}a_3^{(2)}+b_1^{(3)}))^2+\frac{1}{2}(y_2^{(i)}-f(w_{21}^{(3)}a_1^{(2)}+w_{22}^{(3)}a_2^{(2)}+w_{23}^{(3)}a_3^{(2)}+b_2^{(3)}))^2
\end{align}\]
</div>
</div>
<div class="paragraph">
<p>由求导的链式法则，对“输出层神经元的权重参数”求偏导，有：</p>
</div>
<div class="stemblock">
<div class="content">
\[\begin{align}
\frac{\partial E}{\partial w_{11}^{(3)}} &amp;= \frac{1}{2} \cdot 2(y_1-a_1^{(3)})(-\frac{\partial a_1^{(3)}}{\partial w_{11}^{(3)}}) \\
&amp;= -(y_1-a_1^{(3)})f'(z_1^{3})\frac{\partial z_1^{(3)}}{\partial w_{11}^{(3)}} \\
&amp;= -(y_1-a_1^{(3)})f'(z_1^{3})a_1^{(2)}
\end{align}\]
</div>
</div>
</div>
</div>
</div>
</div>
</div>
<div id="footer">
<div id="footer-text">
Version 1.0<br>
Last updated 2019-10-03 20:53:11 +0800
</div>
</div>
<style>
pre.pygments .hll { background-color: #ffffcc }
pre.pygments  { background: #f8f8f8; }
pre.pygments .tok-c { color: #408080; font-style: italic } /* Comment */
pre.pygments .tok-err { border: 1px solid #FF0000 } /* Error */
pre.pygments .tok-k { color: #008000; font-weight: bold } /* Keyword */
pre.pygments .tok-o { color: #666666 } /* Operator */
pre.pygments .tok-ch { color: #408080; font-style: italic } /* Comment.Hashbang */
pre.pygments .tok-cm { color: #408080; font-style: italic } /* Comment.Multiline */
pre.pygments .tok-cp { color: #BC7A00 } /* Comment.Preproc */
pre.pygments .tok-cpf { color: #408080; font-style: italic } /* Comment.PreprocFile */
pre.pygments .tok-c1 { color: #408080; font-style: italic } /* Comment.Single */
pre.pygments .tok-cs { color: #408080; font-style: italic } /* Comment.Special */
pre.pygments .tok-gd { color: #A00000 } /* Generic.Deleted */
pre.pygments .tok-ge { font-style: italic } /* Generic.Emph */
pre.pygments .tok-gr { color: #FF0000 } /* Generic.Error */
pre.pygments .tok-gh { color: #000080; font-weight: bold } /* Generic.Heading */
pre.pygments .tok-gi { color: #00A000 } /* Generic.Inserted */
pre.pygments .tok-go { color: #888888 } /* Generic.Output */
pre.pygments .tok-gp { color: #000080; font-weight: bold } /* Generic.Prompt */
pre.pygments .tok-gs { font-weight: bold } /* Generic.Strong */
pre.pygments .tok-gu { color: #800080; font-weight: bold } /* Generic.Subheading */
pre.pygments .tok-gt { color: #0044DD } /* Generic.Traceback */
pre.pygments .tok-kc { color: #008000; font-weight: bold } /* Keyword.Constant */
pre.pygments .tok-kd { color: #008000; font-weight: bold } /* Keyword.Declaration */
pre.pygments .tok-kn { color: #008000; font-weight: bold } /* Keyword.Namespace */
pre.pygments .tok-kp { color: #008000 } /* Keyword.Pseudo */
pre.pygments .tok-kr { color: #008000; font-weight: bold } /* Keyword.Reserved */
pre.pygments .tok-kt { color: #B00040 } /* Keyword.Type */
pre.pygments .tok-m { color: #666666 } /* Literal.Number */
pre.pygments .tok-s { color: #BA2121 } /* Literal.String */
pre.pygments .tok-na { color: #7D9029 } /* Name.Attribute */
pre.pygments .tok-nb { color: #008000 } /* Name.Builtin */
pre.pygments .tok-nc { color: #0000FF; font-weight: bold } /* Name.Class */
pre.pygments .tok-no { color: #880000 } /* Name.Constant */
pre.pygments .tok-nd { color: #AA22FF } /* Name.Decorator */
pre.pygments .tok-ni { color: #999999; font-weight: bold } /* Name.Entity */
pre.pygments .tok-ne { color: #D2413A; font-weight: bold } /* Name.Exception */
pre.pygments .tok-nf { color: #0000FF } /* Name.Function */
pre.pygments .tok-nl { color: #A0A000 } /* Name.Label */
pre.pygments .tok-nn { color: #0000FF; font-weight: bold } /* Name.Namespace */
pre.pygments .tok-nt { color: #008000; font-weight: bold } /* Name.Tag */
pre.pygments .tok-nv { color: #19177C } /* Name.Variable */
pre.pygments .tok-ow { color: #AA22FF; font-weight: bold } /* Operator.Word */
pre.pygments .tok-w { color: #bbbbbb } /* Text.Whitespace */
pre.pygments .tok-mb { color: #666666 } /* Literal.Number.Bin */
pre.pygments .tok-mf { color: #666666 } /* Literal.Number.Float */
pre.pygments .tok-mh { color: #666666 } /* Literal.Number.Hex */
pre.pygments .tok-mi { color: #666666 } /* Literal.Number.Integer */
pre.pygments .tok-mo { color: #666666 } /* Literal.Number.Oct */
pre.pygments .tok-sa { color: #BA2121 } /* Literal.String.Affix */
pre.pygments .tok-sb { color: #BA2121 } /* Literal.String.Backtick */
pre.pygments .tok-sc { color: #BA2121 } /* Literal.String.Char */
pre.pygments .tok-dl { color: #BA2121 } /* Literal.String.Delimiter */
pre.pygments .tok-sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */
pre.pygments .tok-s2 { color: #BA2121 } /* Literal.String.Double */
pre.pygments .tok-se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */
pre.pygments .tok-sh { color: #BA2121 } /* Literal.String.Heredoc */
pre.pygments .tok-si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */
pre.pygments .tok-sx { color: #008000 } /* Literal.String.Other */
pre.pygments .tok-sr { color: #BB6688 } /* Literal.String.Regex */
pre.pygments .tok-s1 { color: #BA2121 } /* Literal.String.Single */
pre.pygments .tok-ss { color: #19177C } /* Literal.String.Symbol */
pre.pygments .tok-bp { color: #008000 } /* Name.Builtin.Pseudo */
pre.pygments .tok-fm { color: #0000FF } /* Name.Function.Magic */
pre.pygments .tok-vc { color: #19177C } /* Name.Variable.Class */
pre.pygments .tok-vg { color: #19177C } /* Name.Variable.Global */
pre.pygments .tok-vi { color: #19177C } /* Name.Variable.Instance */
pre.pygments .tok-vm { color: #19177C } /* Name.Variable.Magic */
pre.pygments .tok-il { color: #666666 } /* Literal.Number.Integer.Long */
</style>
<script type="text/x-mathjax-config">
MathJax.Hub.Config({
  messageStyle: "none",
  tex2jax: {
    inlineMath: [["\\(", "\\)"]],
    displayMath: [["\\[", "\\]"]],
    ignoreClass: "nostem|nolatexmath"
  },
  asciimath2jax: {
    delimiters: [["\\$", "\\$"]],
    ignoreClass: "nostem|noasciimath"
  },
  TeX: { equationNumbers: { autoNumber: "none" } }
})
MathJax.Hub.Register.StartupHook("AsciiMath Jax Ready", function () {
  MathJax.InputJax.AsciiMath.postfilterHooks.Add(function (data, node) {
    if ((node = data.script.parentNode) && (node = node.parentNode) && node.classList.contains('stemblock')) {
      data.math.root.display = "block"
    }
    return data
  })
})
</script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-MML-AM_HTMLorMML"></script>
</body>
</html>